{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n!pip install timm # install pytorch image models\n!pip install torchmetrics","metadata":{"papermill":{"duration":20.299559,"end_time":"2022-05-15T02:42:06.284782","exception":false,"start_time":"2022-05-15T02:41:45.985223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:52:42.960493Z","iopub.execute_input":"2022-07-03T02:52:42.961139Z","iopub.status.idle":"2022-07-03T02:56:24.654546Z","shell.execute_reply.started":"2022-07-03T02:52:42.961053Z","shell.execute_reply":"2022-07-03T02:56:24.653678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# torch.backends.cuda.matmul.allow_tf32 = True\n# torch.backends.cudnn.benchmark = False\n# torch.backends.cudnn.deterministic = True\n# torch.backends.cudnn.allow_tf32 = True\n# data = torch.randn([8, 304, 16, 16], dtype=torch.half, device='cuda', requires_grad=True)\n# net = torch.nn.Conv2d(304, 1824, kernel_size=[1, 1], padding=[0, 0], stride=[1, 1], dilation=[1, 1], groups=1)\n# net = net.cuda().half()\n# out = net(data)\n# out.backward(torch.randn_like(out))\n# torch.cuda.synchronize()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T02:56:24.657088Z","iopub.execute_input":"2022-07-03T02:56:24.657376Z","iopub.status.idle":"2022-07-03T02:56:24.663270Z","shell.execute_reply.started":"2022-07-03T02:56:24.657338Z","shell.execute_reply":"2022-07-03T02:56:24.662250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport pandas as pd\nimport numpy as np\nimport random \n\nimport albumentations as A\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\n\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom torch import nn\nimport torchmetrics \nfrom torch.nn.modules.loss import _Loss\nfrom  torch.cuda.amp import autocast, GradScaler","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":9.937056,"end_time":"2022-05-15T02:42:16.252097","exception":false,"start_time":"2022-05-15T02:42:06.315041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:24.664861Z","iopub.execute_input":"2022-07-03T02:56:24.665130Z","iopub.status.idle":"2022-07-03T02:56:29.339935Z","shell.execute_reply.started":"2022-07-03T02:56:24.665094Z","shell.execute_reply":"2022-07-03T02:56:29.339147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"papermill":{"duration":0.744504,"end_time":"2022-05-15T02:42:17.026962","exception":false,"start_time":"2022-05-15T02:42:16.282458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:29.342163Z","iopub.execute_input":"2022-07-03T02:56:29.342423Z","iopub.status.idle":"2022-07-03T02:56:30.171649Z","shell.execute_reply.started":"2022-07-03T02:56:29.342389Z","shell.execute_reply":"2022-07-03T02:56:30.170746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalConstantsConfigure():\n    def __init__(self):\n        self.continue_training = False\n        self.last_model = '../input/near-final/tf_efficientnetv2_m_in21k_40_last.pt' \n        self.num_epochs_done = 40\n        self.seed = 127 # 107\n        self.fold = 1\n        self.num_folds = 4\n        self.num_classes = 100\n        self.biggest_loss = 999\n        self.training_size_rate = 0.8\n        self.training_dir = '../input/sorghum-id-fgvc-9/train_images'\n        self.model_name = 'efficientnet_b7'\n        self.model_path = './efficientnet_b7.pt'\n        self.image_size = 256\n        self.batch_size = 32 # 16\n        self.val_batch_size = 256\n        self.lr = 3e-5 # 3e-5\n        self.num_epochs = 5\n        self.steps_per_decay = 5\n        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n        self.num_workers = 1  # if torch.cuda.is_available() else 4\ngcc = GlobalConstantsConfigure()","metadata":{"papermill":{"duration":0.098421,"end_time":"2022-05-15T02:42:17.15626","exception":false,"start_time":"2022-05-15T02:42:17.057839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:30.173021Z","iopub.execute_input":"2022-07-03T02:56:30.173301Z","iopub.status.idle":"2022-07-03T02:56:30.246692Z","shell.execute_reply.started":"2022-07-03T02:56:30.173261Z","shell.execute_reply":"2022-07-03T02:56:30.245871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed) : \n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nset_seed(gcc.seed)","metadata":{"papermill":{"duration":0.044999,"end_time":"2022-05-15T02:42:17.231345","exception":false,"start_time":"2022-05-15T02:42:17.186346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:30.248182Z","iopub.execute_input":"2022-07-03T02:56:30.248739Z","iopub.status.idle":"2022-07-03T02:56:30.520160Z","shell.execute_reply.started":"2022-07-03T02:56:30.248700Z","shell.execute_reply":"2022-07-03T02:56:30.519362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')\nprint(len(df_all))\ndf_all.dropna(inplace=True)\nprint(len(df_all))\ndf_all.head()","metadata":{"papermill":{"duration":0.097813,"end_time":"2022-05-15T02:42:17.359361","exception":false,"start_time":"2022-05-15T02:42:17.261548","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:30.521343Z","iopub.execute_input":"2022-07-03T02:56:30.521592Z","iopub.status.idle":"2022-07-03T02:56:30.609671Z","shell.execute_reply.started":"2022-07-03T02:56:30.521561Z","shell.execute_reply":"2022-07-03T02:56:30.608981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_cultivars = list(df_all[\"cultivar\"].unique())","metadata":{"papermill":{"duration":0.044312,"end_time":"2022-05-15T02:42:17.43483","exception":false,"start_time":"2022-05-15T02:42:17.390518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:30.610918Z","iopub.execute_input":"2022-07-03T02:56:30.611310Z","iopub.status.idle":"2022-07-03T02:56:30.620763Z","shell.execute_reply.started":"2022-07-03T02:56:30.611275Z","shell.execute_reply":"2022-07-03T02:56:30.619947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"file_path\"] = df_all[\"image\"].apply(lambda image: '../input/sorghum-id-fgvc-9/train_images/' + image)\ndf_all[\"cultivar_index\"] = df_all[\"cultivar\"].map(lambda item: unique_cultivars.index(item))\ndf_all[\"is_exist\"] = df_all[\"file_path\"].apply(lambda file_path: os.path.exists(file_path))\ndf_all = df_all[df_all.is_exist==True]\ndf_all.head()","metadata":{"papermill":{"duration":41.392566,"end_time":"2022-05-15T02:42:58.858163","exception":false,"start_time":"2022-05-15T02:42:17.465597","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:56:30.622346Z","iopub.execute_input":"2022-07-03T02:56:30.622783Z","iopub.status.idle":"2022-07-03T02:57:01.046917Z","shell.execute_reply.started":"2022-07-03T02:56:30.622742Z","shell.execute_reply":"2022-07-03T02:57:01.046171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=gcc.num_folds, shuffle=True, random_state=gcc.seed)\n\n\ntrain_folds = []\nval_folds = []\n\nfor train_idx, valid_idx in skf.split(df_all['image'], df_all[\"cultivar_index\"]):\n    train_folds.append(train_idx)\n    val_folds.append(valid_idx)\n#     df_train = df_all.iloc[train_idx]\n#     df_valid = df_all.iloc[valid_idx]\n\n# print(train_folds)\n# print(val_folds)\ndf_train = df_all.iloc[train_folds[gcc.fold]]\ndf_valid = df_all.iloc[val_folds[gcc.fold]]\n\n\n\n\nprint(f\"train size: {len(df_train)}\")\nprint(f\"valid size: {len(df_valid)}\")\n\nprint(df_train.cultivar.value_counts())\nprint(df_valid.cultivar.value_counts())","metadata":{"papermill":{"duration":0.068452,"end_time":"2022-05-15T02:42:58.958295","exception":false,"start_time":"2022-05-15T02:42:58.889843","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.050125Z","iopub.execute_input":"2022-07-03T02:57:01.050343Z","iopub.status.idle":"2022-07-03T02:57:01.079531Z","shell.execute_reply.started":"2022-07-03T02:57:01.050301Z","shell.execute_reply":"2022-07-03T02:57:01.078779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import rcParams\nrcParams['figure.figsize'] = 20,40\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int64(W * cut_rat)\n    cut_h = np.int64(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.3,0.4)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    new_data = data.clone()\n    new_data[:, :, bby1:bby2, bbx1:bbx2] = data[indices, :, bby1:bby2, bbx1:bbx2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return new_data, targets\n","metadata":{"papermill":{"duration":0.056354,"end_time":"2022-05-15T02:42:59.051932","exception":false,"start_time":"2022-05-15T02:42:58.995578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.080813Z","iopub.execute_input":"2022-07-03T02:57:01.081061Z","iopub.status.idle":"2022-07-03T02:57:01.095420Z","shell.execute_reply.started":"2022-07-03T02:57:01.081028Z","shell.execute_reply":"2022-07-03T02:57:01.094743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumDataset(Dataset):\n    def __init__(self, dirs, labels, transformation=None):\n        super(SorghumDataset,self).__init__()\n        self.dirs = dirs\n        self.labels = labels\n        self.transformation = transformation\n    def __len__(self):\n        return len(self.dirs)\n\n    def __getitem__(self, index):\n        image = cv2.imread(self.dirs[index])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        label = self.labels[index] # need to one hot encoding here\n        \n        image = np.array(image)\n\n        if self.transformation:\n            aug_image = self.transformation(image=image)\n            image = aug_image['image']\n            \n        image = image / 255.\n        image = image.transpose((2, 0, 1))\n        \n        image = torch.from_numpy(image).type(torch.float32)\n        image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n        \n        labels = torch.from_numpy(np.array(self.labels[index])).type(torch.float32)\n\n\n        return image, labels","metadata":{"papermill":{"duration":0.043394,"end_time":"2022-05-15T02:42:59.129655","exception":false,"start_time":"2022-05-15T02:42:59.086261","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.096920Z","iopub.execute_input":"2022-07-03T02:57:01.097455Z","iopub.status.idle":"2022-07-03T02:57:01.116725Z","shell.execute_reply.started":"2022-07-03T02:57:01.097419Z","shell.execute_reply":"2022-07-03T02:57:01.115749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0),\n    A.Flip(p=0.5),\n    A.RandomRotate90(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.HueSaturationValue(p=0.5),\n#     A.OneOf([\n#         A.RandomBrightnessContrast(p=0.5),\n#         A.RandomGamma(p=0.5),\n#     ], p=0.5),\n#     A.OneOf([\n#         A.Blur(p=0.1),\n#         A.GaussianBlur(p=0.1),\n#         A.MotionBlur(p=0.1),\n#     ], p=0.1),\n#     A.OneOf([\n#         A.GaussNoise(p=0.1),\n#         A.ISONoise(p=0.1),\n#         A.GridDropout(ratio=0.5, p=0.2),\n#         A.CoarseDropout(max_holes=16, min_holes=8, max_height=16, max_width=16, min_height=8, min_width=8, p=0.2)\n#     ], p=0.2),\n\n])\nvalidation_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0)\n])","metadata":{"papermill":{"duration":0.040413,"end_time":"2022-05-15T02:42:59.201618","exception":false,"start_time":"2022-05-15T02:42:59.161205","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.118114Z","iopub.execute_input":"2022-07-03T02:57:01.118711Z","iopub.status.idle":"2022-07-03T02:57:01.131244Z","shell.execute_reply.started":"2022-07-03T02:57:01.118674Z","shell.execute_reply":"2022-07-03T02:57:01.130306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_set = SorghumDataset(training_dirs, training_labels, training_transformation)\n# validation_set = SorghumDataset(validation_dirs, validation_labels, validation_transformation)\n\ntraining_set = SorghumDataset(df_train.file_path.values, df_train.cultivar_index.values, training_transformation)\nvalidation_set = SorghumDataset(df_valid.file_path.values, df_valid.cultivar_index.values, validation_transformation)\n\n\ntraining_dataloader = DataLoader(\n    training_set,\n    batch_size = gcc.batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True, \n    drop_last = True\n)\nvalidation_dataloader = DataLoader(\n    validation_set,\n    batch_size = gcc.val_batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True,\n    drop_last = True\n)","metadata":{"papermill":{"duration":0.040182,"end_time":"2022-05-15T02:42:59.27266","exception":false,"start_time":"2022-05-15T02:42:59.232478","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.132794Z","iopub.execute_input":"2022-07-03T02:57:01.133382Z","iopub.status.idle":"2022-07-03T02:57:01.144370Z","shell.execute_reply.started":"2022-07-03T02:57:01.133346Z","shell.execute_reply":"2022-07-03T02:57:01.143367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(torch.nn.Module): \n    def __init__(self, model_backbone):\n        super(CustomModel,self).__init__()\n        self.model = model_backbone\n        \n        self.model.classifier = nn.Sequential(\n            nn.BatchNorm1d(2560),\n            nn.Linear(2560, 512),\n            nn.Dropout(0.5),\n            nn.ReLU(inplace=True),\n            nn.Linear(512, gcc.num_classes),\n            \n#             nn.BatchNorm1d(1280),\n#             nn.Linear(1280, 512),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n\n#             nn.Linear(512, 256),\n#             nn.Dropout(0.5),\n#             nn.SiLU(inplace=True),\n#             nn.Linear(256, gcc.num_classes)\n        )\n    def forward(self,x):\n        x = self.model(x)\n        return x\n","metadata":{"papermill":{"duration":0.039676,"end_time":"2022-05-15T02:42:59.343183","exception":false,"start_time":"2022-05-15T02:42:59.303507","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.151597Z","iopub.execute_input":"2022-07-03T02:57:01.152107Z","iopub.status.idle":"2022-07-03T02:57:01.166237Z","shell.execute_reply.started":"2022-07-03T02:57:01.152073Z","shell.execute_reply":"2022-07-03T02:57:01.164358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_one_hot(labels, num_classes, dtype=torch.float, dim=1):\n    if labels.ndim < dim + 1:\n        shape = list(labels.shape) + [1] * (dim + 1 - len(labels.shape))\n        labels = torch.reshape(labels, shape)\n    sh = list(labels.shape)\n    sh[dim] = num_classes\n    o = torch.zeros(size=sh, dtype=dtype, device=labels.device)\n    labels = o.scatter_(dim=dim, index=labels.long(), value=1)\n    return labels\n\n\nclass PolyLoss(_Loss):\n    def __init__(self, softmax, ce_weight=None, reduction='mean', epsilon=1.0):\n        super().__init__()\n        self.softmax = softmax\n        self.reduction = reduction\n        self.epsilon = epsilon\n        self.cross_entropy = nn.CrossEntropyLoss(weight=ce_weight, reduction='none')\n\n    def forward(self, input, target):\n\n        if len(input.shape) - len(target.shape) == 1:\n            target = target.unsqueeze(1).long()\n        n_pred_ch, n_target_ch = input.shape[1], target.shape[1]\n        if n_pred_ch != n_target_ch:\n            self.ce_loss = self.cross_entropy(input, torch.squeeze(target, dim=1).long())\n            target = to_one_hot(target, num_classes=n_pred_ch)\n        else:\n            self.ce_loss = self.cross_entropy(input, torch.argmax(target, dim=1))\n\n        if self.softmax:\n            input = torch.softmax(input, 1)\n\n        pt = (input * target).sum(dim=1) \n        \n        poly_loss = self.ce_loss + self.epsilon * (1 - pt)\n\n        polyl = torch.mean(poly_loss)  # the batch and channel average\n        # polyl = torch.sum(poly_loss)  # sum over the batch and channel dims\n        return (polyl)","metadata":{"papermill":{"duration":0.045133,"end_time":"2022-05-15T02:42:59.419914","exception":false,"start_time":"2022-05-15T02:42:59.374781","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.168807Z","iopub.execute_input":"2022-07-03T02:57:01.169263Z","iopub.status.idle":"2022-07-03T02:57:01.199510Z","shell.execute_reply.started":"2022-07-03T02:57:01.169226Z","shell.execute_reply":"2022-07-03T02:57:01.198559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# backbone = models.efficientnet_b5(pretrained=True) \nbackbone = timm.create_model(gcc.model_name,pretrained=True)\n\n\n# print(index)\nscaler = GradScaler()   \nmodel = CustomModel(backbone)\n# ce = torch.nn.CrossEntropyLoss()\n# loss_func = PolyLoss(ce) # torch.nn.CrossEntropyLoss()\nloss_func = torch.nn.CrossEntropyLoss().to(gcc.device)\nmetrics_acc = torchmetrics.Accuracy(threshold=0.0, num_classes = gcc.num_classes)\nprint(model)\n\n\n# for index, child in enumerate(backbone.children()):\n#     print(index)\n#     if index <= 7:\n#         for param in child.parameters():\n#             param.requires_grad = False\n\n\ntrainable_parameters = [param for param in model.parameters() if param.requires_grad == True]\noptimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n# optimizer = torch.optim.SGD(trainable_parameters, lr = gcc.lr, momentum = 0.9)\n# optimizer = torch.optim.SGD(trainable_parameters, lr = gcc.lr, momentum=0.9)\n# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2)\n\nmodel.to(gcc.device)\n\nif gcc.continue_training == True:\n    # model.load_state_dict(torch.load(gcc.last_model))\n    checkpoint = torch.load(gcc.last_model)\n    \n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    \n    # optimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n    # optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n    \n    # lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, gcc.steps_per_decay)\n    # lr_scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n    \n    # print(lr_scheduler.state_dict())\n\nprint('load model done')","metadata":{"papermill":{"duration":29.563898,"end_time":"2022-05-15T02:43:29.015116","exception":false,"start_time":"2022-05-15T02:42:59.451218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:01.200834Z","iopub.execute_input":"2022-07-03T02:57:01.201248Z","iopub.status.idle":"2022-07-03T02:57:05.641192Z","shell.execute_reply.started":"2022-07-03T02:57:01.201214Z","shell.execute_reply":"2022-07-03T02:57:05.640407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def calc_accuracy(pred, true):\n#     # print(pred, true)\n#     true = true.type(torch.int64) # label\n#     pred = F.softmax(pred, dim = 1)\n#     true = torch.zeros(pred.shape[0], pred.shape[1]).scatter_(1, true.unsqueeze(1), 1.)\n#     acc = (true.argmax(-1) == pred.argmax(-1)).float().detach().numpy()\n#     acc = float(acc.sum() / len(acc))\n#     return round(acc, 4)","metadata":{"papermill":{"duration":0.040453,"end_time":"2022-05-15T02:43:29.089759","exception":false,"start_time":"2022-05-15T02:43:29.049306","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:05.642643Z","iopub.execute_input":"2022-07-03T02:57:05.643167Z","iopub.status.idle":"2022-07-03T02:57:05.646922Z","shell.execute_reply.started":"2022-07-03T02:57:05.643129Z","shell.execute_reply":"2022-07-03T02:57:05.646275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_progress(training_dataloader, loss_func, scheduler):\n    model.train()\n    training_loss = 0\n    training_acc = 0\n    cnt = 0 \n    print('Learning rate: ',scheduler.get_last_lr())\n    print(scheduler.state_dict())\n    \n    scaler = torch.cuda.amp.GradScaler()\n    \n    training_loader = tqdm(training_dataloader, desc='Iterating through the training set')\n    for image, label in training_loader:\n        image = image.to(gcc.device).float()\n        label = label.to(gcc.device).long()\n        \n        mix_decision = np.random.rand()\n        if mix_decision < 0.25:\n            image, label = cutmix(image, label, 1.)\n\n        # with autocast():\n        with torch.cuda.amp.autocast(): \n            output = model(image.float())\n\n            if mix_decision < 0.25:\n\n                # acc = metrics_acc(output, label[0].cpu().argmax(1)) * label[2].cpu().argmax(1) + metric_acc(output, label[1].cpu().argmax(1)) * (1. - label[2].cpu().argmax(1)) # metrics_acc(output.cpu().argmax(1), label.cpu().int())\n                loss = loss_func(output, label[0]) * label[2] + loss_func(output, label[1]) * (1. - label[2])\n                \n            else:\n                loss = loss_func(output, label)\n\n\n        acc = 0 # metrics_acc(output.cpu().argmax(1), label.cpu().int())\n        # loss = loss_func(output, label.long())\n\n        training_loss += loss.detach().item()\n        training_acc += acc\n        cnt +=1 \n        \n        optimizer.zero_grad()\n        # loss.backward()\n        # optimizer.step()\n        \n        scaler.scale(loss).backward() \n        scaler.step(optimizer) \n        scaler.update()\n        \n        scheduler.step()\n    \n    mean_training_loss = training_loss / cnt\n    mean_training_acc = training_acc / cnt\n    \n    return mean_training_loss, mean_training_acc\n    ","metadata":{"papermill":{"duration":0.04628,"end_time":"2022-05-15T02:43:29.169155","exception":false,"start_time":"2022-05-15T02:43:29.122875","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:05.648590Z","iopub.execute_input":"2022-07-03T02:57:05.648981Z","iopub.status.idle":"2022-07-03T02:57:05.664927Z","shell.execute_reply.started":"2022-07-03T02:57:05.648947Z","shell.execute_reply":"2022-07-03T02:57:05.664156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validation_progress(validation_dataloader, loss_func):\n    model.eval()\n    validation_loss = 0\n    validation_acc = 0\n    cnt = 0 \n    validation_loader = tqdm(validation_dataloader, desc='Iterating through the validation set')\n    with torch.no_grad():\n        with torch.cuda.amp.autocast():\n            for image, label in validation_loader:\n                image = image.to(gcc.device)\n                label = label.to(gcc.device)\n\n                output = model(image)\n                loss = loss_func(output, label.long())\n                # acc = calc_accuracy(output.cpu(), label.cpu())\n                # output.to(gcc.device)\n                acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n                # calculate accuracy here\n                validation_loss += loss.detach().item()\n                validation_acc += acc\n\n                cnt += 1\n\n    mean_validation_loss = validation_loss / cnt\n    mean_validation_acc = validation_acc / cnt\n    return mean_validation_loss, mean_validation_acc","metadata":{"papermill":{"duration":0.042979,"end_time":"2022-05-15T02:43:29.24513","exception":false,"start_time":"2022-05-15T02:43:29.202151","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:05.666403Z","iopub.execute_input":"2022-07-03T02:57:05.666648Z","iopub.status.idle":"2022-07-03T02:57:05.680884Z","shell.execute_reply.started":"2022-07-03T02:57:05.666616Z","shell.execute_reply":"2022-07-03T02:57:05.680002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_model(model, training_dataloader, validation_dataloader, loss_func, scheduler):\n    training_losses_history, validation_losses_history = [], []\n    training_acc_history, validation_acc_history = [], []\n    best_loss = gcc.biggest_loss\n    for epoch in range(gcc.num_epochs):\n        \n        training_loss, training_acc = training_progress(training_dataloader, loss_func, scheduler)\n        training_losses_history.append(training_loss)\n        training_acc_history.append(training_acc)\n        \n        validation_loss, validation_acc = validation_progress(validation_dataloader, loss_func)\n        validation_losses_history.append(validation_loss)\n        validation_acc_history.append(validation_acc)\n        \n        if validation_loss <= best_loss: # sussy baka\n            best_loss = validation_loss\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_best.pt')\n            # torch.save(model.state_dict(), gcc.model_name + '_best.pt')\n        \n        if epoch == gcc.num_epochs - 1: # i believe my timing capability\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_' + str(gcc.num_epochs_done + gcc.num_epochs) + '_last.pt')\n        \n        print(f'Epoch {epoch + 1}/{gcc.num_epochs} | Training_loss : {training_loss:.3f} | Validation_loss : {validation_loss:.3f}' \n             + f' Training_acc : {training_acc:.3f} | Validation_acc : {validation_acc:.3f}'\n             )\n        torch.cuda.empty_cache()\n    return training_losses_history, validation_losses_history, training_acc_history, validation_acc_history\n","metadata":{"papermill":{"duration":0.045573,"end_time":"2022-05-15T02:43:29.324245","exception":false,"start_time":"2022-05-15T02:43:29.278672","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:05.682213Z","iopub.execute_input":"2022-07-03T02:57:05.682663Z","iopub.status.idle":"2022-07-03T02:57:05.695393Z","shell.execute_reply.started":"2022-07-03T02:57:05.682627Z","shell.execute_reply":"2022-07-03T02:57:05.694664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses_history, validation_losses_history, training_acc_history, validation_acc_history = training_model(model, training_dataloader, validation_dataloader, loss_func, lr_scheduler)","metadata":{"papermill":{"duration":32460.236904,"end_time":"2022-05-15T11:44:29.594253","exception":false,"start_time":"2022-05-15T02:43:29.357349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-03T02:57:05.696552Z","iopub.execute_input":"2022-07-03T02:57:05.696988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing_progress(testing_dataloader, loss_func)","metadata":{"papermill":{"duration":15.599587,"end_time":"2022-05-15T11:44:59.771867","exception":false,"start_time":"2022-05-15T11:44:44.17228","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_history(model_name, train_loss_history, val_loss_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_loss_history, label='Train Loss', lw=3)\n    plt.plot(x, val_loss_history, label='Validation Loss', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Loss\", fontsize=15)\n\n    plt.show()\n    \nplot_loss_history(gcc.model_name, training_losses_history, validation_losses_history, gcc.num_epochs)","metadata":{"papermill":{"duration":15.355875,"end_time":"2022-05-15T11:45:29.737309","exception":false,"start_time":"2022-05-15T11:45:14.381434","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_acc_history(model_name, train_acc_history, val_acc_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_acc_history, label='Training Accuracy', lw=3)\n    plt.plot(x, val_acc_history, label='Validation Accuracy', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Accuracy\", fontsize=15)\n\n    plt.show()\n    \nplot_acc_history(gcc.model_name, training_acc_history, validation_acc_history, gcc.num_epochs)","metadata":{"papermill":{"duration":15.266458,"end_time":"2022-05-15T11:45:59.867503","exception":false,"start_time":"2022-05-15T11:45:44.601045","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(gcc.model_name + '_best.pt')\nmodel.load_state_dict(checkpoint['model_state_dict'])","metadata":{"papermill":{"duration":16.577703,"end_time":"2022-05-15T11:46:30.941925","exception":false,"start_time":"2022-05-15T11:46:14.364222","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub.head()","metadata":{"papermill":{"duration":15.05354,"end_time":"2022-05-15T11:47:00.512858","exception":false,"start_time":"2022-05-15T11:46:45.459318","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"filename\"] = sub[\"filename\"].apply(lambda image: '../input/sorghum-id-fgvc-9/test/' + image)\nsub[\"cultivar\"] = 0\nsub.head()","metadata":{"papermill":{"duration":14.592959,"end_time":"2022-05-15T11:47:29.938039","exception":false,"start_time":"2022-05-15T11:47:15.34508","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testing_dataset = SorghumDataset(sub['filename'], sub['cultivar'], validation_transformation)\ntesting_dataloader = DataLoader(testing_dataset, \n                                batch_size=gcc.val_batch_size, \n                                shuffle=False, \n                                num_workers=gcc.num_workers)","metadata":{"papermill":{"duration":14.557059,"end_time":"2022-05-15T11:47:59.440386","exception":false,"start_time":"2022-05-15T11:47:44.883327","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.zeros(len(testing_dataloader))\npredictions = []\ncnt = 0 \nwith torch.no_grad():\n    for image, label in tqdm(testing_dataloader):\n        image = image.to(gcc.device)\n        outputs = model(image)\n        # print(outputs)\n        preds = outputs.detach().cpu()\n        predictions.append(preds.argmax(1)) # need optimize here\n        # print(predictions)","metadata":{"papermill":{"duration":1234.674931,"end_time":"2022-05-15T12:08:49.430358","exception":false,"start_time":"2022-05-15T11:48:14.755427","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = predictions[0]\nfor i in range(len(predictions) - 1):\n    tmp = torch.cat((tmp, predictions[i+1]))","metadata":{"papermill":{"duration":14.585764,"end_time":"2022-05-15T12:09:19.303547","exception":false,"start_time":"2022-05-15T12:09:04.717783","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = label_encoder.inverse_transform(tmp)\npredictions = [unique_cultivars[pred] for pred in tmp]","metadata":{"papermill":{"duration":15.264096,"end_time":"2022-05-15T12:09:49.13467","exception":false,"start_time":"2022-05-15T12:09:33.870574","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub['cultivar'] = predictions\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"papermill":{"duration":15.270386,"end_time":"2022-05-15T12:10:18.808727","exception":false,"start_time":"2022-05-15T12:10:03.538341","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}