{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n!pip install timm # install pytorch image models\n!pip install torchmetrics","metadata":{"papermill":{"duration":20.299559,"end_time":"2022-05-15T02:42:06.284782","exception":false,"start_time":"2022-05-15T02:41:45.985223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:06.774372Z","iopub.execute_input":"2022-06-04T05:37:06.774708Z","iopub.status.idle":"2022-06-04T05:37:24.064911Z","shell.execute_reply.started":"2022-06-04T05:37:06.774677Z","shell.execute_reply":"2022-06-04T05:37:24.064041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# torch.backends.cuda.matmul.allow_tf32 = True\n# torch.backends.cudnn.benchmark = False\n# torch.backends.cudnn.deterministic = True\n# torch.backends.cudnn.allow_tf32 = True\n# data = torch.randn([8, 304, 16, 16], dtype=torch.half, device='cuda', requires_grad=True)\n# net = torch.nn.Conv2d(304, 1824, kernel_size=[1, 1], padding=[0, 0], stride=[1, 1], dilation=[1, 1], groups=1)\n# net = net.cuda().half()\n# out = net(data)\n# out.backward(torch.randn_like(out))\n# torch.cuda.synchronize()","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:37:24.068502Z","iopub.execute_input":"2022-06-04T05:37:24.068713Z","iopub.status.idle":"2022-06-04T05:37:24.072904Z","shell.execute_reply.started":"2022-06-04T05:37:24.068686Z","shell.execute_reply":"2022-06-04T05:37:24.071983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport pandas as pd\nimport numpy as np\nimport random \n\nimport albumentations as A\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\n\nimport torchvision\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torch.nn.functional as F\nfrom torch import nn\nimport torchmetrics \nfrom torch.nn.modules.loss import _Loss\nfrom  torch.cuda.amp import autocast, GradScaler","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":9.937056,"end_time":"2022-05-15T02:42:16.252097","exception":false,"start_time":"2022-05-15T02:42:06.315041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:24.074180Z","iopub.execute_input":"2022-06-04T05:37:24.074691Z","iopub.status.idle":"2022-06-04T05:37:29.155133Z","shell.execute_reply.started":"2022-06-04T05:37:24.074653Z","shell.execute_reply":"2022-06-04T05:37:29.154284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"papermill":{"duration":0.744504,"end_time":"2022-05-15T02:42:17.026962","exception":false,"start_time":"2022-05-15T02:42:16.282458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:29.157743Z","iopub.execute_input":"2022-06-04T05:37:29.158184Z","iopub.status.idle":"2022-06-04T05:37:29.897511Z","shell.execute_reply.started":"2022-06-04T05:37:29.158141Z","shell.execute_reply":"2022-06-04T05:37:29.896709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GlobalConstantsConfigure():\n    def __init__(self):\n        self.continue_training = True\n        self.last_model = '../input/sorghum-100-cultivar-identification/resnext50d_32x4d_60_last.pt' \n        self.num_epochs_done = 40\n        self.seed = 127 # 107\n        self.fold = 1\n        self.num_folds = 4\n        self.num_classes = 100\n        self.biggest_loss = 999\n        self.training_size_rate = 0.8\n        self.training_dir = '../input/sorghum-id-fgvc-9/train_images'\n        self.model_name = 'efficientnet_b3'\n        self.model_path = './rexnet_150.pt'\n        self.image_size = 512\n        self.batch_size = 8\n        self.val_batch_size = 32\n        self.lr = 3e-5 # 3e-5\n        self.num_epochs = 20\n        self.steps_per_decay = 5\n        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n        self.num_workers = 1  # if torch.cuda.is_available() else 4\ngcc = GlobalConstantsConfigure()","metadata":{"papermill":{"duration":0.098421,"end_time":"2022-05-15T02:42:17.15626","exception":false,"start_time":"2022-05-15T02:42:17.057839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:29.900722Z","iopub.execute_input":"2022-06-04T05:37:29.900978Z","iopub.status.idle":"2022-06-04T05:37:29.972473Z","shell.execute_reply.started":"2022-06-04T05:37:29.900951Z","shell.execute_reply":"2022-06-04T05:37:29.971704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#timm.list_models(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:37:29.975366Z","iopub.execute_input":"2022-06-04T05:37:29.977420Z","iopub.status.idle":"2022-06-04T05:37:29.983092Z","shell.execute_reply.started":"2022-06-04T05:37:29.977348Z","shell.execute_reply":"2022-06-04T05:37:29.982262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed) : \n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seed(gcc.seed)","metadata":{"papermill":{"duration":0.044999,"end_time":"2022-05-15T02:42:17.231345","exception":false,"start_time":"2022-05-15T02:42:17.186346","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:29.985051Z","iopub.execute_input":"2022-06-04T05:37:29.985519Z","iopub.status.idle":"2022-06-04T05:37:29.994667Z","shell.execute_reply.started":"2022-06-04T05:37:29.985483Z","shell.execute_reply":"2022-06-04T05:37:29.993934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')\nprint(len(df_all))\ndf_all.dropna(inplace=True)\nprint(len(df_all))\ndf_all.head()","metadata":{"papermill":{"duration":0.097813,"end_time":"2022-05-15T02:42:17.359361","exception":false,"start_time":"2022-05-15T02:42:17.261548","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:29.996075Z","iopub.execute_input":"2022-06-04T05:37:29.996567Z","iopub.status.idle":"2022-06-04T05:37:30.056027Z","shell.execute_reply.started":"2022-06-04T05:37:29.996531Z","shell.execute_reply":"2022-06-04T05:37:30.055343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_cultivars = list(df_all[\"cultivar\"].unique())","metadata":{"papermill":{"duration":0.044312,"end_time":"2022-05-15T02:42:17.43483","exception":false,"start_time":"2022-05-15T02:42:17.390518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:30.057322Z","iopub.execute_input":"2022-06-04T05:37:30.057766Z","iopub.status.idle":"2022-06-04T05:37:30.067842Z","shell.execute_reply.started":"2022-06-04T05:37:30.057731Z","shell.execute_reply":"2022-06-04T05:37:30.067156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"file_path\"] = df_all[\"image\"].apply(lambda image: '../input/sorghum-id-fgvc-9/train_images/' + image)\ndf_all[\"cultivar_index\"] = df_all[\"cultivar\"].map(lambda item: unique_cultivars.index(item))\ndf_all[\"is_exist\"] = df_all[\"file_path\"].apply(lambda file_path: os.path.exists(file_path))\ndf_all = df_all[df_all.is_exist==True]\ndf_all.head()","metadata":{"papermill":{"duration":41.392566,"end_time":"2022-05-15T02:42:58.858163","exception":false,"start_time":"2022-05-15T02:42:17.465597","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:37:30.072435Z","iopub.execute_input":"2022-06-04T05:37:30.072986Z","iopub.status.idle":"2022-06-04T05:38:08.307488Z","shell.execute_reply.started":"2022-06-04T05:37:30.072958Z","shell.execute_reply":"2022-06-04T05:38:08.306721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=gcc.num_folds, shuffle=True, random_state=gcc.seed)\n\n\ntrain_folds = []\nval_folds = []\n\nfor train_idx, valid_idx in skf.split(df_all['image'], df_all[\"cultivar_index\"]):\n    train_folds.append(train_idx)\n    val_folds.append(valid_idx)\n#     df_train = df_all.iloc[train_idx]\n#     df_valid = df_all.iloc[valid_idx]\n\n# print(train_folds)\n# print(val_folds)\ndf_train = df_all.iloc[train_folds[gcc.fold]]\ndf_valid = df_all.iloc[val_folds[gcc.fold]]\n\n\n\n\nprint(f\"train size: {len(df_train)}\")\nprint(f\"valid size: {len(df_valid)}\")\n\nprint(df_train.cultivar.value_counts())\nprint(df_valid.cultivar.value_counts())","metadata":{"papermill":{"duration":0.068452,"end_time":"2022-05-15T02:42:58.958295","exception":false,"start_time":"2022-05-15T02:42:58.889843","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.308837Z","iopub.execute_input":"2022-06-04T05:38:08.309077Z","iopub.status.idle":"2022-06-04T05:38:08.339448Z","shell.execute_reply.started":"2022-06-04T05:38:08.309043Z","shell.execute_reply":"2022-06-04T05:38:08.338746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pylab import rcParams\nrcParams['figure.figsize'] = 20,40\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int64(W * cut_rat)\n    cut_h = np.int64(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\ndef cutmix(data, target, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha),0.3,0.4)\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    new_data = data.clone()\n    new_data[:, :, bby1:bby2, bbx1:bbx2] = data[indices, :, bby1:bby2, bbx1:bbx2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return new_data, targets\n","metadata":{"papermill":{"duration":0.056354,"end_time":"2022-05-15T02:42:59.051932","exception":false,"start_time":"2022-05-15T02:42:58.995578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.340548Z","iopub.execute_input":"2022-06-04T05:38:08.340777Z","iopub.status.idle":"2022-06-04T05:38:08.516258Z","shell.execute_reply.started":"2022-06-04T05:38:08.340741Z","shell.execute_reply":"2022-06-04T05:38:08.515326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumDataset(Dataset):\n    def __init__(self, dirs, labels, transformation=None):\n        super(SorghumDataset,self).__init__()\n        self.dirs = dirs\n        self.labels = labels\n        self.transformation = transformation\n    def __len__(self):\n        return len(self.dirs)\n\n    def __getitem__(self, index):\n        image = cv2.imread(self.dirs[index])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        label = self.labels[index] # need to one hot encoding here\n        \n        image = np.array(image)\n\n        if self.transformation:\n            aug_image = self.transformation(image=image)\n            image = aug_image['image']\n            \n        image = image / 255.\n        image = image.transpose((2, 0, 1))\n        \n        image = torch.from_numpy(image).type(torch.float32)\n        image = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))(image)\n        \n        labels = torch.from_numpy(np.array(self.labels[index])).type(torch.float32)\n\n\n        return image, labels","metadata":{"papermill":{"duration":0.043394,"end_time":"2022-05-15T02:42:59.129655","exception":false,"start_time":"2022-05-15T02:42:59.086261","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.517712Z","iopub.execute_input":"2022-06-04T05:38:08.518226Z","iopub.status.idle":"2022-06-04T05:38:08.527342Z","shell.execute_reply.started":"2022-06-04T05:38:08.518188Z","shell.execute_reply":"2022-06-04T05:38:08.526662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0),\n    A.Flip(p=0.5),\n    A.RandomRotate90(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.HueSaturationValue(p=0.5),\n\n])\nvalidation_transformation = A.Compose([\n    A.Resize(width=gcc.image_size, height=gcc.image_size, p=1.0)\n])","metadata":{"papermill":{"duration":0.040413,"end_time":"2022-05-15T02:42:59.201618","exception":false,"start_time":"2022-05-15T02:42:59.161205","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.528589Z","iopub.execute_input":"2022-06-04T05:38:08.529061Z","iopub.status.idle":"2022-06-04T05:38:08.539760Z","shell.execute_reply.started":"2022-06-04T05:38:08.529026Z","shell.execute_reply":"2022-06-04T05:38:08.539039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training_set = SorghumDataset(training_dirs, training_labels, training_transformation)\n# validation_set = SorghumDataset(validation_dirs, validation_labels, validation_transformation)\n\ntraining_set = SorghumDataset(df_train.file_path.values, df_train.cultivar_index.values, training_transformation)\nvalidation_set = SorghumDataset(df_valid.file_path.values, df_valid.cultivar_index.values, validation_transformation)\n\n\ntraining_dataloader = DataLoader(\n    training_set,\n    batch_size = gcc.batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True, \n    drop_last = True\n)\n\nvalidation_dataloader = DataLoader(\n    validation_set,\n    batch_size = gcc.val_batch_size,\n    shuffle = True,\n    num_workers = gcc.num_workers,\n    pin_memory = True,\n    drop_last = True\n)","metadata":{"papermill":{"duration":0.040182,"end_time":"2022-05-15T02:42:59.27266","exception":false,"start_time":"2022-05-15T02:42:59.232478","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.540955Z","iopub.execute_input":"2022-06-04T05:38:08.541624Z","iopub.status.idle":"2022-06-04T05:38:08.549734Z","shell.execute_reply.started":"2022-06-04T05:38:08.541588Z","shell.execute_reply":"2022-06-04T05:38:08.549099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(torch.nn.Module): \n    def __init__(self, model_backbone):\n        super(CustomModel,self).__init__()\n        self.model = model_backbone\n        self.num_in_features = self.model.get_classifier().in_features\n        print(self.num_in_features)\n        self.model.classifier = nn.Sequential(\n            nn.BatchNorm1d(self.num_in_features),\n            nn.Linear(self.num_in_features, 512),\n            nn.Dropout(0.5),\n            nn.ReLU(inplace=True),\n            nn.Linear(512, gcc.num_classes),\n        )\n    def forward(self,x):\n        x = self.model(x)\n        return x\n","metadata":{"papermill":{"duration":0.039676,"end_time":"2022-05-15T02:42:59.343183","exception":false,"start_time":"2022-05-15T02:42:59.303507","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.551003Z","iopub.execute_input":"2022-06-04T05:38:08.551475Z","iopub.status.idle":"2022-06-04T05:38:08.560683Z","shell.execute_reply.started":"2022-06-04T05:38:08.551432Z","shell.execute_reply":"2022-06-04T05:38:08.559912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_one_hot(labels, num_classes, dtype=torch.float, dim=1):\n    if labels.ndim < dim + 1:\n        shape = list(labels.shape) + [1] * (dim + 1 - len(labels.shape))\n        labels = torch.reshape(labels, shape)\n    sh = list(labels.shape)\n    sh[dim] = num_classes\n    o = torch.zeros(size=sh, dtype=dtype, device=labels.device)\n    labels = o.scatter_(dim=dim, index=labels.long(), value=1)\n    return labels\n\n\nclass PolyLoss(_Loss):\n    def __init__(self, softmax, ce_weight=None, reduction='mean', epsilon=1.0):\n        super().__init__()\n        self.softmax = softmax\n        self.reduction = reduction\n        self.epsilon = epsilon\n        self.cross_entropy = nn.CrossEntropyLoss(weight=ce_weight, reduction='none')\n\n    def forward(self, input, target):\n\n        if len(input.shape) - len(target.shape) == 1:\n            target = target.unsqueeze(1).long()\n        n_pred_ch, n_target_ch = input.shape[1], target.shape[1]\n        if n_pred_ch != n_target_ch:\n            self.ce_loss = self.cross_entropy(input, torch.squeeze(target, dim=1).long())\n            target = to_one_hot(target, num_classes=n_pred_ch)\n        else:\n            self.ce_loss = self.cross_entropy(input, torch.argmax(target, dim=1))\n\n        if self.softmax:\n            input = torch.softmax(input, 1)\n\n        pt = (input * target).sum(dim=1) \n        \n        poly_loss = self.ce_loss + self.epsilon * (1 - pt)\n\n        polyl = torch.mean(poly_loss)  # the batch and channel average\n        # polyl = torch.sum(poly_loss)  # sum over the batch and channel dims\n        return (polyl)","metadata":{"papermill":{"duration":0.045133,"end_time":"2022-05-15T02:42:59.419914","exception":false,"start_time":"2022-05-15T02:42:59.374781","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.561948Z","iopub.execute_input":"2022-06-04T05:38:08.562680Z","iopub.status.idle":"2022-06-04T05:38:08.576451Z","shell.execute_reply.started":"2022-06-04T05:38:08.562644Z","shell.execute_reply":"2022-06-04T05:38:08.575719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # backbone = models.efficientnet_b5(pretrained=True) \n# backbone = timm.create_model(gcc.model_name,pretrained=True)\n\n\n# # print(index)\n# scaler = GradScaler()   \n# model = CustomModel(backbone)\n# # ce = torch.nn.CrossEntropyLoss()\n# # loss_func = PolyLoss(ce) # torch.nn.CrossEntropyLoss()\n# loss_func = torch.nn.CrossEntropyLoss().to(gcc.device)\n# metrics_acc = torchmetrics.Accuracy(threshold=0.0, num_classes = gcc.num_classes)\n\n# trainable_parameters = [param for param in model.parameters() if param.requires_grad == True]\n# optimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n# lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2)\n\n# model.to(gcc.device)\n\n# if gcc.continue_training == True:\n#     checkpoint = torch.load(gcc.last_model)\n    \n#     model.load_state_dict(checkpoint['model_state_dict'])\n\n# print('load model done')","metadata":{"papermill":{"duration":29.563898,"end_time":"2022-05-15T02:43:29.015116","exception":false,"start_time":"2022-05-15T02:42:59.451218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:38:08.577776Z","iopub.execute_input":"2022-06-04T05:38:08.578604Z","iopub.status.idle":"2022-06-04T05:38:18.687960Z","shell.execute_reply.started":"2022-06-04T05:38:08.578568Z","shell.execute_reply":"2022-06-04T05:38:18.685452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# backbone = models.efficientnet_b3(pretrained=True) \nbackbone = timm.create_model(gcc.model_name,pretrained=True)\n\n# print(index)\nscaler = GradScaler()   \nmodel = CustomModel(backbone)\n# ce = torch.nn.CrossEntropyLoss()\n# loss_func = PolyLoss(ce) # torch.nn.CrossEntropyLoss()\nloss_func = torch.nn.CrossEntropyLoss().to(gcc.device)\nmetrics_acc = torchmetrics.Accuracy(threshold=0.0, num_classes = gcc.num_classes)\n# print(model)\n\n\n# for index, child in enumerate(backbone.children()):\n#     print(index)\n#     if index <= 7:\n#         for param in child.parameters():\n#             param.requires_grad = False\n\n\ntrainable_parameters = [param for param in model.parameters() if param.requires_grad == True]\noptimizer = torch.optim.Adam(trainable_parameters, lr = gcc.lr)\n\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=5, T_mult=2)\n\nmodel.to(gcc.device)\n\nprint('load model done')","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:40:44.871555Z","iopub.execute_input":"2022-06-04T05:40:44.871870Z","iopub.status.idle":"2022-06-04T05:40:45.224474Z","shell.execute_reply.started":"2022-06-04T05:40:44.871837Z","shell.execute_reply":"2022-06-04T05:40:45.223704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as T\n\ntensor = torch.rand(3,8,8)\ntransform = T.ToPILImage()\nimg = transform(tensor)\nimg.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:40:47.577325Z","iopub.execute_input":"2022-06-04T05:40:47.578008Z","iopub.status.idle":"2022-06-04T05:40:47.643498Z","shell.execute_reply.started":"2022-06-04T05:40:47.577970Z","shell.execute_reply":"2022-06-04T05:40:47.642397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_set[0]","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:41:00.487064Z","iopub.execute_input":"2022-06-04T05:41:00.487344Z","iopub.status.idle":"2022-06-04T05:41:00.622359Z","shell.execute_reply.started":"2022-06-04T05:41:00.487314Z","shell.execute_reply":"2022-06-04T05:41:00.621655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #training_loader = tqdm(training_dataloader, desc='Iterating through the training set')\n# #training_set = SorghumDataset(df_train.file_path.values, df_train.cultivar_index.values, training_transformation)\n# #for image, label in training_loader:\n# #    image = image.to(gcc.device).float()\n# #    label = label.to(gcc.device).long()\n# #    \n# #    mix_decision = np.random.rand()\n# #    if mix_decision < 0.25:\n# image = (training_set[1][0]).to(gcc.device).float()\n# label = (training_set[1][1]).to(gcc.device).long()\n# print(label)\n# mix_decision = np.random.rand()\n\n# if mix_decision < 0.25:\n#     image, label = cutmix(image, label, 1.)\n\n# #image, label = cutmix(image, label, 1.)","metadata":{"execution":{"iopub.status.busy":"2022-06-04T05:41:03.212701Z","iopub.execute_input":"2022-06-04T05:41:03.213280Z","iopub.status.idle":"2022-06-04T05:41:03.218560Z","shell.execute_reply.started":"2022-06-04T05:41:03.213242Z","shell.execute_reply":"2022-06-04T05:41:03.217152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as T\n\nfrom torchvision.transforms.functional import to_pil_image\n\ndef training_progress(training_dataloader, loss_func, scheduler):\n    model.train()\n    training_loss = 0\n    training_acc = 0\n    cnt = 0 \n    print('Learning rate: ',scheduler.get_last_lr())\n    print(scheduler.state_dict())\n    training_loader = tqdm(training_dataloader, desc='Iterating through the training set')\n    for image, label in training_loader:\n        image = image.to(gcc.device).float()\n        label = label.to(gcc.device).long()\n#         print(image[0])\n\n#         plt.figure()\n#         plt.subplot(1,2,1)\n#         plt.imshow(to_pil_image(image[0]), cmap='gray')\n#         plt.title('train')\n\n        mix_decision = np.random.rand()\n        if mix_decision < 0.25:\n            image, label = cutmix(image, label, 1.)\n\n        with autocast():\n            output = model(image.float())\n\n            if mix_decision < 0.25:\n\n                # acc = metrics_acc(output, label[0].cpu().argmax(1)) * label[2].cpu().argmax(1) + metric_acc(output, label[1].cpu().argmax(1)) * (1. - label[2].cpu().argmax(1)) # metrics_acc(output.cpu().argmax(1), label.cpu().int())\n                loss = loss_func(output, label[0]) * label[2] + loss_func(output, label[1]) * (1. - label[2])\n                \n            else:\n                loss = loss_func(output, label)\n\n\n        acc = 0 # metrics_acc(output.cpu().argmax(1), label.cpu().int())\n        # loss = loss_func(output, label.long())\n\n        training_loss += loss.detach().item()\n        training_acc += acc\n        cnt +=1 \n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n    \n    mean_training_loss = training_loss / cnt\n    mean_training_acc = training_acc / cnt\n    \n    return mean_training_loss, mean_training_acc\n    ","metadata":{"papermill":{"duration":0.04628,"end_time":"2022-05-15T02:43:29.169155","exception":false,"start_time":"2022-05-15T02:43:29.122875","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:41:04.045337Z","iopub.execute_input":"2022-06-04T05:41:04.045607Z","iopub.status.idle":"2022-06-04T05:41:04.056688Z","shell.execute_reply.started":"2022-06-04T05:41:04.045577Z","shell.execute_reply":"2022-06-04T05:41:04.055713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list = []\n\ndef validation_progress(validation_dataloader, loss_func):\n    model.eval()\n    validation_loss = 0\n    validation_acc = 0\n    cnt = 0 \n    global label_list\n    validation_loader = tqdm(validation_dataloader, desc='Iterating through the validation set')\n    with torch.no_grad():\n        for image, label in validation_loader:\n            image = image.to(gcc.device)\n            label = label.to(gcc.device)\n\n            output = model(image)\n            loss = loss_func(output, label.long())\n            # acc = calc_accuracy(output.cpu(), label.cpu())\n            # output.to(gcc.device)\n            acc = metrics_acc(output.cpu().argmax(1), label.cpu().int())\n            label_list.append(output.cpu().argmax(1))\n            label_list.append(label.cpu().int())\n            # calculate accuracy here\n            validation_loss += loss.detach().item()\n            validation_acc += acc\n            \n            cnt += 1\n\n    mean_validation_loss = validation_loss / cnt\n    mean_validation_acc = validation_acc / cnt\n    return mean_validation_loss, mean_validation_acc","metadata":{"papermill":{"duration":0.042979,"end_time":"2022-05-15T02:43:29.24513","exception":false,"start_time":"2022-05-15T02:43:29.202151","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:41:04.263441Z","iopub.execute_input":"2022-06-04T05:41:04.263773Z","iopub.status.idle":"2022-06-04T05:41:04.272696Z","shell.execute_reply.started":"2022-06-04T05:41:04.263741Z","shell.execute_reply":"2022-06-04T05:41:04.271861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_model(model, training_dataloader, validation_dataloader, loss_func, scheduler):\n    training_losses_history, validation_losses_history = [], []\n    training_acc_history, validation_acc_history = [], []\n    best_loss = gcc.biggest_loss\n    for epoch in range(gcc.num_epochs):\n        \n        training_loss, training_acc = training_progress(training_dataloader, loss_func, scheduler)\n        training_losses_history.append(training_loss)\n        training_acc_history.append(training_acc)\n        \n        validation_loss, validation_acc = validation_progress(validation_dataloader, loss_func)\n        validation_losses_history.append(validation_loss)\n        validation_acc_history.append(validation_acc)\n        \n        if validation_loss <= best_loss: # sussy baka\n            best_loss = validation_loss\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_best.pt')\n            # torch.save(model.state_dict(), gcc.model_name + '_best.pt')\n        \n        if epoch == gcc.num_epochs - 1: # i believe my timing capability\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler_state_dict': scheduler.state_dict()\n            }, gcc.model_name + '_' + str(gcc.num_epochs_done + gcc.num_epochs) + '_last.pt')\n\n        print(f'Epoch {epoch + 1}/{gcc.num_epochs} | Training_loss : {training_loss:.3f} | Validation_loss : {validation_loss:.3f}' \n             + f' Training_acc : {training_acc:.3f} | Validation_acc : {validation_acc:.3f}'\n             )\n    return training_losses_history, validation_losses_history, training_acc_history, validation_acc_history\n","metadata":{"papermill":{"duration":0.045573,"end_time":"2022-05-15T02:43:29.324245","exception":false,"start_time":"2022-05-15T02:43:29.278672","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:41:04.405930Z","iopub.execute_input":"2022-06-04T05:41:04.406383Z","iopub.status.idle":"2022-06-04T05:41:04.418058Z","shell.execute_reply.started":"2022-06-04T05:41:04.406350Z","shell.execute_reply":"2022-06-04T05:41:04.417327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses_history, validation_losses_history, training_acc_history, validation_acc_history = training_model(model, training_dataloader, validation_dataloader, loss_func, lr_scheduler)","metadata":{"papermill":{"duration":32460.236904,"end_time":"2022-05-15T11:44:29.594253","exception":false,"start_time":"2022-05-15T02:43:29.357349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-04T05:41:04.562323Z","iopub.execute_input":"2022-06-04T05:41:04.564365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing_progress(testing_dataloader, loss_func)","metadata":{"papermill":{"duration":15.599587,"end_time":"2022-05-15T11:44:59.771867","exception":false,"start_time":"2022-05-15T11:44:44.17228","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss_history(model_name, train_loss_history, val_loss_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_loss_history, label='Train Loss', lw=3)\n    plt.plot(x, val_loss_history, label='Validation Loss', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Loss\", fontsize=15)\n\n    plt.show()\n    \nplot_loss_history(gcc.model_name, training_losses_history, validation_losses_history, gcc.num_epochs)","metadata":{"papermill":{"duration":15.355875,"end_time":"2022-05-15T11:45:29.737309","exception":false,"start_time":"2022-05-15T11:45:14.381434","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_acc_history(model_name, train_acc_history, val_acc_history, num_epochs):\n    \n    x = np.arange(num_epochs)\n    fig = plt.figure(figsize=(10, 6))\n    plt.plot(x, train_acc_history, label='Training Accuracy', lw=3)\n    plt.plot(x, val_acc_history, label='Validation Accuracy', lw=3)\n\n    plt.title(f\"{model_name}\", fontsize=20)\n    plt.legend(fontsize=12)\n    plt.xlabel(\"Epoch\", fontsize=15)\n    plt.ylabel(\"Accuracy\", fontsize=15)\n\n    plt.show()\n    \nplot_acc_history(gcc.model_name, training_acc_history, validation_acc_history, gcc.num_epochs)","metadata":{"papermill":{"duration":15.266458,"end_time":"2022-05-15T11:45:59.867503","exception":false,"start_time":"2022-05-15T11:45:44.601045","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = torch.load(gcc.model_name + '_best.pt')\nmodel.load_state_dict(checkpoint['model_state_dict'])","metadata":{"papermill":{"duration":16.577703,"end_time":"2022-05-15T11:46:30.941925","exception":false,"start_time":"2022-05-15T11:46:14.364222","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub.head()","metadata":{"papermill":{"duration":15.05354,"end_time":"2022-05-15T11:47:00.512858","exception":false,"start_time":"2022-05-15T11:46:45.459318","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[\"filename\"] = sub[\"filename\"].apply(lambda image: '../input/sorghum-id-fgvc-9/test/' + image)\nsub[\"cultivar\"] = 0\nsub.head()","metadata":{"papermill":{"duration":14.592959,"end_time":"2022-05-15T11:47:29.938039","exception":false,"start_time":"2022-05-15T11:47:15.34508","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testing_dataset = SorghumDataset(sub['filename'], sub['cultivar'], validation_transformation)\ntesting_dataloader = DataLoader(testing_dataset, \n                                batch_size=gcc.val_batch_size, \n                                shuffle=False, \n                                num_workers=gcc.num_workers)","metadata":{"papermill":{"duration":14.557059,"end_time":"2022-05-15T11:47:59.440386","exception":false,"start_time":"2022-05-15T11:47:44.883327","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.zeros(len(testing_dataloader))\npredictions = []\ncnt = 0 \nwith torch.no_grad():\n    for image, label in tqdm(testing_dataloader):\n        image = image.to(gcc.device)\n        outputs = model(image)\n        # print(outputs)\n        preds = outputs.detach().cpu()\n        predictions.append(preds.argmax(1)) # need optimize here\n        # print(predictions)","metadata":{"papermill":{"duration":1234.674931,"end_time":"2022-05-15T12:08:49.430358","exception":false,"start_time":"2022-05-15T11:48:14.755427","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = predictions[0]\nfor i in range(len(predictions) - 1):\n    tmp = torch.cat((tmp, predictions[i+1]))","metadata":{"papermill":{"duration":14.585764,"end_time":"2022-05-15T12:09:19.303547","exception":false,"start_time":"2022-05-15T12:09:04.717783","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = label_encoder.inverse_transform(tmp)\npredictions = [unique_cultivars[pred] for pred in tmp]","metadata":{"papermill":{"duration":15.264096,"end_time":"2022-05-15T12:09:49.13467","exception":false,"start_time":"2022-05-15T12:09:33.870574","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\nsub['cultivar'] = predictions\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"papermill":{"duration":15.270386,"end_time":"2022-05-15T12:10:18.808727","exception":false,"start_time":"2022-05-15T12:10:03.538341","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}