{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is my implementation of EfficientNetV2-M.","metadata":{}},{"cell_type":"markdown","source":"This notebook is inspired by the article: https://arxiv.org/pdf/2104.00298.pdf\n","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom PIL import Image\nimport os\nimport time\nimport copy\n\nimport torch\nfrom torch.optim import lr_scheduler\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom pytorch_lightning.metrics import Metric\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.319025Z","iopub.execute_input":"2021-06-03T15:02:38.319433Z","iopub.status.idle":"2021-06-03T15:02:38.32594Z","shell.execute_reply.started":"2021-06-03T15:02:38.319401Z","shell.execute_reply":"2021-06-03T15:02:38.324677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nsys.path.append('../input/torchcontrib/contrib-master/')\nimport torchcontrib\nfrom torchcontrib.optim import SWA\nfrom torch.optim.swa_utils import AveragedModel, SWALR","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.333865Z","iopub.execute_input":"2021-06-03T15:02:38.334857Z","iopub.status.idle":"2021-06-03T15:02:38.341003Z","shell.execute_reply.started":"2021-06-03T15:02:38.334811Z","shell.execute_reply":"2021-06-03T15:02:38.339803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"package_paths = ['../input/timm-pytorch-image-models/pytorch-image-models-master',]\nimport sys;\nfor pth in package_paths:\n    sys.path.append(pth)\n# load the external python package\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.34336Z","iopub.execute_input":"2021-06-03T15:02:38.344289Z","iopub.status.idle":"2021-06-03T15:02:38.354202Z","shell.execute_reply.started":"2021-06-03T15:02:38.344239Z","shell.execute_reply":"2021-06-03T15:02:38.352525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since my model is pretrained on 21k ImageNet and finetuned on  ILSVRC2012, i'm applying only drop out and stochastic depth with data augmentation. Weight decay disabled. \n","metadata":{}},{"cell_type":"code","source":"\"\"\"\nInitialize parameters\n\"\"\"\nclass CFG:\n    # directories\n    TRAIN_CSV_PATH = '../input/plant-pathology-2021-fgvc8/train.csv'\n    TRAIN_DIR = '../input/plant-pathology-resized/train_640'#to train faster use already resized images\n    TEST_DIR = '../input/plant-pathology-2021-fgvc8/test_images'\n    PRETRAINED_DIR = '../input/efficientnetv2/Efficient_net18'#directory for pretrained model\n    MODELDIR = '../output/EfficientNet'#directory for saving model\n    SWADIR = '../output/SWA'#directory for saving SWA model\n    # data info\n    label_num2str = {0: 'powdery_mildew',\n                     1: 'scab',\n                     2: 'complex',\n                     3: 'frog_eye_leaf_spot',\n                     4: 'rust'}\n    \n    label_str2num = {'powdery_mildew': 0,\n                     'scab': 1,\n                     'complex': 2,\n                     'frog_eye_leaf_spot': 3,\n                     'rust': 4}\n    valnum=3700\n    trainnum=14800\n    num_classes = 5\n    \n    # parameters for training\n    TRAIN = False\n    THRESHOLD = False # turn on for tuning threshold\n    BATCH = 6 # small batches working as a kind of regularization that improves generalization ability\n    EPOCHS = 30\n    WEIGHT_DECAY = 0.00000# disabled\n    LR = 5e-4\n    min_LR = 1e-7\n    T_MAX = EPOCHS-1\n    IM_SIZE = 640\n    DPR = 0.2# Drop path rate a.k.a. Stochastic depth\n    DR = 0.5# Drop out rate\n    \n    # SWA\n    use_swa = False\n    swa_lr = 5e-5\n    swa_start = 18\n    \n\n    DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n\n    # training hyper-parameters\n    fl_alpha = 1.0  # alpha of focal_loss\n    fl_gamma = 2.0  # gamma of focal_loss\n    cls_weight = [3.6480, 1.0000, 2.1840, 1.5001, 2.2901] # class weights for calculation of loss on train\n    class_weights = torch.tensor([5.4663, 1.0223, 2.3602, 1.494, 2.5669, 1],  device=DEVICE)# class weights for validation of threshold, where distribution slightly different\n    threshold = torch.tensor([0.3750, 0.4750, 0.3750, 0.4650, 0.4450], device=DEVICE)# calculated on tuning\n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.35753Z","iopub.execute_input":"2021-06-03T15:02:38.358494Z","iopub.status.idle":"2021-06-03T15:02:38.375093Z","shell.execute_reply.started":"2021-06-03T15:02:38.358265Z","shell.execute_reply":"2021-06-03T15:02:38.373488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nRead data\n\"\"\"\ntrain_df = pd.read_csv(CFG.TRAIN_CSV_PATH) \ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.37838Z","iopub.execute_input":"2021-06-03T15:02:38.379479Z","iopub.status.idle":"2021-06-03T15:02:38.429612Z","shell.execute_reply.started":"2021-06-03T15:02:38.379435Z","shell.execute_reply":"2021-06-03T15:02:38.428285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nAnalyze distribtion to calculate weight of each class\n\"\"\"\ntrain_df['labels'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.431789Z","iopub.execute_input":"2021-06-03T15:02:38.432209Z","iopub.status.idle":"2021-06-03T15:02:38.445342Z","shell.execute_reply.started":"2021-06-03T15:02:38.432166Z","shell.execute_reply":"2021-06-03T15:02:38.44352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are three ways to fight class imbalance:\n1. Downsampling - delete some elements to make equal distribution\n2. Upsampling - add new elements with data augmentation to make equal distribution.\n3. Weighting - use different weight to each class, so that smaller classes will have proportionately more weight.\n\nThe best way is to use 2nd method, but it requires more computations, so i choosed 3rd way. This method may lead to unstable learning step, because every batch will have different amount of \"virtual\" elements. \n","metadata":{}},{"cell_type":"code","source":"\"\"\"\nAdd numerical labels for dataframe\n\"\"\"\nall_numeric_labels = []\nfor row_idx, row in train_df.iterrows():\n    labels_list = row['labels'].split(\" \")\n    numeric_label_list = [CFG.label_str2num[each] for each in labels_list if each != 'healthy']\n    all_numeric_labels.append(numeric_label_list)\ntrain_df['numerical labels'] = all_numeric_labels\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:38.448054Z","iopub.execute_input":"2021-06-03T15:02:38.448726Z","iopub.status.idle":"2021-06-03T15:02:40.593324Z","shell.execute_reply.started":"2021-06-03T15:02:38.448652Z","shell.execute_reply":"2021-06-03T15:02:40.592142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nCreate class to get transformed data\n\"\"\"\nclass GetData(Dataset):\n    def __init__(self, Dir, FNames: list, Labels: list, Transform):\n        self.dir = Dir\n        self.fnames = FNames\n        self.transform = Transform\n        self.labels = Labels \n    \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):       \n        x = Image.open(os.path.join(self.dir, self.fnames[index]))\n    \n        if \"train\" in self.dir: \n            return self.transform(x), self.labels[index]\n        elif \"test\" in self.dir: \n            return self.transform(x), self.fnames[index]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.595483Z","iopub.execute_input":"2021-06-03T15:02:40.596248Z","iopub.status.idle":"2021-06-03T15:02:40.606106Z","shell.execute_reply.started":"2021-06-03T15:02:40.596201Z","shell.execute_reply":"2021-06-03T15:02:40.604798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nCreate list of image names and labels\n\"\"\"\nall_img_names: list = train_df[\"image\"].values.tolist()\nall_img_labels: list = train_df[\"numerical labels\"].values.tolist()\nall_img_labels_ts = []\nfor tmp_lb in all_img_labels:\n    tmp_label = np.zeros([CFG.num_classes])\n    for idx in tmp_lb:\n        tmp_label[idx] = 1.0\n    all_img_labels_ts.append(tmp_label)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.608353Z","iopub.execute_input":"2021-06-03T15:02:40.608933Z","iopub.status.idle":"2021-06-03T15:02:40.646129Z","shell.execute_reply.started":"2021-06-03T15:02:40.608886Z","shell.execute_reply":"2021-06-03T15:02:40.6448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nCreate train and validation data \n\"\"\"\nX_Train = all_img_names[:CFG.trainnum]\nY_Train = all_img_labels_ts[:CFG.trainnum]\nX_val = all_img_names[-CFG.valnum:]\nY_val = all_img_labels_ts[-CFG.valnum:]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.647995Z","iopub.execute_input":"2021-06-03T15:02:40.648671Z","iopub.status.idle":"2021-06-03T15:02:40.658484Z","shell.execute_reply.started":"2021-06-03T15:02:40.64862Z","shell.execute_reply":"2021-06-03T15:02:40.65733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nTransform training data\n\"\"\"\nTransform = transforms.Compose(\n    [transforms.RandomCrop(CFG.IM_SIZE*0.8),\n   #transforms.RandomApply([transforms.ColorJitter((0,0.3), (0,0.3), (0,0.3)),transforms.RandomPerspective(distortion_scale=(0.15)),], p=0.3),    \n   #transforms.RandomApply([transforms.ColorJitter((0,0.3), (0,0.3), (0,0.3)),transforms.RandomAffine(degrees=15),], p=0.3),\n   #transforms.RandomVerticalFlip(p=0.3),   \n   #transforms.RandomHorizontalFlip(p=0.3),\n    transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0),\n    transforms.ToTensor(),\n    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.660793Z","iopub.execute_input":"2021-06-03T15:02:40.661116Z","iopub.status.idle":"2021-06-03T15:02:40.673117Z","shell.execute_reply.started":"2021-06-03T15:02:40.661072Z","shell.execute_reply":"2021-06-03T15:02:40.672084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = GetData(CFG.TRAIN_DIR, X_Train, Y_Train, Transform)\ntrainloader = DataLoader(trainset, batch_size=CFG.BATCH, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.680446Z","iopub.execute_input":"2021-06-03T15:02:40.680823Z","iopub.status.idle":"2021-06-03T15:02:40.689767Z","shell.execute_reply.started":"2021-06-03T15:02:40.68079Z","shell.execute_reply":"2021-06-03T15:02:40.688559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Transformval = transforms.Compose(\n    [transforms.ToTensor(),\n    transforms.CenterCrop(CFG.IM_SIZE*0.8),\n    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.692723Z","iopub.execute_input":"2021-06-03T15:02:40.693341Z","iopub.status.idle":"2021-06-03T15:02:40.700926Z","shell.execute_reply.started":"2021-06-03T15:02:40.693267Z","shell.execute_reply":"2021-06-03T15:02:40.699621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valset = GetData(CFG.TRAIN_DIR, X_val, Y_val, Transformval)\nvalloader = DataLoader(valset, batch_size=CFG.BATCH, shuffle=False)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.703208Z","iopub.execute_input":"2021-06-03T15:02:40.703794Z","iopub.status.idle":"2021-06-03T15:02:40.712288Z","shell.execute_reply.started":"2021-06-03T15:02:40.703749Z","shell.execute_reply":"2021-06-03T15:02:40.711184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"next(iter(trainloader))[0].shape","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:40.714512Z","iopub.execute_input":"2021-06-03T15:02:40.714907Z","iopub.status.idle":"2021-06-03T15:02:41.055138Z","shell.execute_reply.started":"2021-06-03T15:02:40.714861Z","shell.execute_reply":"2021-06-03T15:02:41.054117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine Focal-Loss\n\"\"\"\n\nclass FocalLoss(nn.Module):\n    \"\"\"\n    The focal loss for fighting against class-imbalance\n    \"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.epsilon = 1e-12  # prevent training from Nan-loss error\n        self.cls_weights = torch.tensor([CFG.cls_weight],dtype=torch.float, requires_grad=False, device=CFG.DEVICE)\n\n    def forward(self, logits, target):\n        \"\"\"\n        logits & target should be tensors with shape [batch_size, num_classes]\n        \"\"\"\n        probs = torch.sigmoid(logits)\n        one_subtract_probs = 1.0 - probs\n        # add epsilon\n        probs_new = probs + self.epsilon\n        one_subtract_probs_new = one_subtract_probs + self.epsilon\n        # calculate focal loss\n        log_pt = target * torch.log(probs_new) + (1.0 - target) * torch.log(one_subtract_probs_new)\n        pt = torch.exp(log_pt)\n        focal_loss = -1.0 * (self.alpha * (1 - pt) ** self.gamma) * log_pt\n        focal_loss = focal_loss * self.cls_weights\n        return torch.mean(focal_loss), probs\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:41.058745Z","iopub.execute_input":"2021-06-03T15:02:41.059029Z","iopub.status.idle":"2021-06-03T15:02:41.069204Z","shell.execute_reply.started":"2021-06-03T15:02:41.059Z","shell.execute_reply":"2021-06-03T15:02:41.067624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nDefine F1 score metric\n\"\"\"\nclass MyF1Score(Metric):\n    def __init__(self, cfg, threshold = CFG.threshold, dist_sync_on_step=False):\n        super().__init__(dist_sync_on_step=dist_sync_on_step)\n        self.cfg = cfg\n        self.threshold = threshold\n        self.add_state(\"tp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fp\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n        self.add_state(\"fn\", default=torch.tensor(0), dist_reduce_fx=\"sum\")\n\n    def update(self, preds: torch.Tensor, target: torch.Tensor):\n        assert preds.shape == target.shape\n        preds_str_batch = self.num_to_str(torch.sigmoid(preds))\n        target_str_batch = self.num_to_str(target)\n        tp, fp, fn = 0, 0, 0\n        for pred_str_list, target_str_list in zip(preds_str_batch, target_str_batch):\n            for pred_str in pred_str_list:\n                if pred_str in target_str_list:\n                    tp += 1\n                if pred_str not in target_str_list:\n                    fp += 1\n\n            for target_str in target_str_list:\n                if target_str not in pred_str_list:\n                    fn += 1\n        self.tp += tp\n        self.fp += fp\n        self.fn += fn\n\n    def compute(self):\n        f1 = 2.0 * self.tp / (2.0 * self.tp + self.fn + self.fp)\n        return f1\n    \n    def num_to_str(self, ts: torch.Tensor) -> list:\n        batch_bool_list = (ts > self.threshold).detach().cpu().numpy().tolist()\n        batch_str_list = []\n        for one_sample_bool in batch_bool_list:\n            lb_str_list = [self.cfg.label_num2str[lb_idx] for lb_idx, bool_val in enumerate(one_sample_bool) if bool_val]\n            if len(lb_str_list) == 0:\n                lb_str_list = ['healthy']\n            batch_str_list.append(lb_str_list)\n        return batch_str_list","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:41.071586Z","iopub.execute_input":"2021-06-03T15:02:41.072531Z","iopub.status.idle":"2021-06-03T15:02:41.090362Z","shell.execute_reply.started":"2021-06-03T15:02:41.072487Z","shell.execute_reply":"2021-06-03T15:02:41.089191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nCreate function that counts correct predictions\n\"\"\"\ndef predictions(running_corrects, probs):\n    predictions = probs > CFG.threshold\n    for batch in range(len(predictions)):\n        if torch.sum(predictions[batch] == labels.data[batch]) == CFG.num_classes:\n            running_corrects+=1\n\n    return running_corrects","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:41.092189Z","iopub.execute_input":"2021-06-03T15:02:41.092695Z","iopub.status.idle":"2021-06-03T15:02:41.107259Z","shell.execute_reply.started":"2021-06-03T15:02:41.09265Z","shell.execute_reply":"2021-06-03T15:02:41.1059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nInitialize model\n\"\"\"\nmodel = timm.create_model('tf_efficientnetv2_m_21ft1k', pretrained=False, drop_path_rate=CFG.DPR, drop_rate= CFG.DR)\n\n#for param in model.parameters():\n#    param.requires_grad = False\n# Parameters of newly constructed modules have requires_grad=True by default\n\n\nmodel.classifier = nn.Linear(1280, CFG.num_classes, bias=True)\nmodel.load_state_dict(torch.load(os.path.join(CFG.PRETRAINED_DIR)))\nmodel.cuda()\n\ncriterion = FocalLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG.LR)\nexp_lr_scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max = CFG.T_MAX, eta_min=CFG.min_LR, last_epoch=-1, verbose=True)\nscheduler = exp_lr_scheduler\nmetric = MyF1Score(CFG)\n\nif CFG.use_swa:\n    swa_model = AveragedModel(model)\n    #swa_model.load_state_dict(torch.load(os.path.join('./model','SWA.pth')))\n    swa_model.cuda()\n    swa_scheduler = SWALR(optimizer, swa_lr=CFG.swa_lr)\n    print(\"SWA initialised\")","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:41.109256Z","iopub.execute_input":"2021-06-03T15:02:41.109778Z","iopub.status.idle":"2021-06-03T15:02:48.193936Z","shell.execute_reply.started":"2021-06-03T15:02:41.109732Z","shell.execute_reply":"2021-06-03T15:02:48.192699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"markdown","source":"In process of training i applied half-precision(FP16), that increases training speed and decreases memory usage.","metadata":{}},{"cell_type":"code","source":"if CFG.TRAIN:\n    since = time.time()\n\n    best_acc = 0.00\n    best_f1 = 0.00\n    best_model_wts = copy.deepcopy(model.state_dict())\n    scaler = torch.cuda.amp.GradScaler() \n    autocast = torch.cuda.amp.autocast()\n\n    for epoch in range(CFG.EPOCHS):\n\n        print('Epoch {}'.format(epoch))\n        print('-' * 10)\n\n        f1 = 0\n        f1_running = 0\n        running_corrects = 0\n\n        #training\n        model = model.train()\n        running_loss = 0.0\n        running_corrects = 0\n        iterations = int((CFG.trainnum + CFG.BATCH - 1) / CFG.BATCH)\n        m = 1\n        optimizer.zero_grad()\n        for i, (images, labels) in enumerate(trainloader):        \n            images = images.to(CFG.DEVICE, dtype=torch.int64)\n            labels = labels.to(CFG.DEVICE, dtype=torch.int64)       \n\n\n            print(f\"Training {m}/{iterations}\", end=\"\\r\")   \n            m += 1\n\n            # Runs the forward pass with autocasting.\n            with torch.set_grad_enabled(True) and autocast:\n                outputs = model(images.float()) \n                loss, probs = criterion(outputs, labels)#Variable(outputs), Variable(labels)\n                f1_running += float(metric(outputs, labels))     \n\n            # Scales loss.  Calls backward() on scaled loss to create scaled gradients.\n            # Backward passes under autocast are not recommended.\n            # Backward ops run in the same dtype autocast chose for corresponding forward ops.\n            scaler.scale(loss).backward()\n\n\n            # scaler.step() first unscales the gradients of the optimizer's assigned params.\n            # If these gradients do not contain infs or NaNs, optimizer.step() is then called,\n            # otherwise, optimizer.step() is skipped.\n            scaler.step(optimizer)\n\n\n            # Updates the scale for next iteration.\n            scaler.update()\n            optimizer.zero_grad()\n\n\n\n            running_loss += loss.item() * images.size(0)\n            running_corrects = predictions(running_corrects, probs)\n\n        f1= f1_running / iterations    \n        epoch_loss = running_loss / len(trainset)\n        epoch_acc = running_corrects / len(trainset)\n        print('Training Loss: {:.6f} Acc: {:.4f}, F1: {:.4f} '.format(\n        epoch_loss, epoch_acc, f1))\n\n\n\n        #validation\n        model.eval()\n\n        f1 = 0\n        f1_running = 0\n        running_loss = 0.0\n        running_corrects = 0\n\n        iterations = int((CFG.valnum + CFG.BATCH-1) / CFG.BATCH)\n        m = 1\n\n        for i, (images, labels) in enumerate(valloader):\n            images = images.to(CFG.DEVICE, dtype=torch.int64)\n            labels = labels.to(CFG.DEVICE, dtype=torch.int64)\n\n            print(f\"Validating {m}/{iterations}\", end=\"\\r\")   \n            m += 1\n\n            with torch.set_grad_enabled(False) and autocast:\n                outputs = model(images.float())\n                loss, probs = criterion(outputs, labels)\n                f1_running += float(metric(outputs, labels))\n\n\n            # statistics\n            running_loss += loss.item() * images.size(0)\n            running_corrects = predictions(running_corrects, probs)\n\n\n        f1= f1_running / iterations\n        epoch_loss = running_loss / len(valset)\n        epoch_acc = running_corrects / len(valset)\n        time_elapsed = time.time() - since\n        print('Validation Loss: {:.6f} Acc: {:.4f} F1:{:.4f}'.format(\n        epoch_loss, epoch_acc, f1))\n        print('Training of epoch {} completed in {:.0f}m {:.0f}s'.format(\n                epoch, time_elapsed // 60, time_elapsed % 60))             \n        #save best model\n        if f1 > best_f1:\n            best_f1 = f1\n            best_model_wts = copy.deepcopy(model.state_dict())\n            torch.save(model.state_dict(),os.path.join(CFG.MODELDIR))\n\n        \n        if epoch >= CFG.swa_start and CFG.use_swa:\n            swa_model.update_parameters(model)\n            swa_scheduler.step()\n            print(\"SWA model update\")\n        else:\n            scheduler.step()\n\n        print()\n\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best validation F1: {:.4f}'.format(best_f1))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)  \n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:48.195732Z","iopub.execute_input":"2021-06-03T15:02:48.196456Z","iopub.status.idle":"2021-06-03T15:02:48.207876Z","shell.execute_reply.started":"2021-06-03T15:02:48.19641Z","shell.execute_reply":"2021-06-03T15:02:48.206569Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After training of SWA model you should do forward propagation to update statistics for BatchNorm layers.\n","metadata":{}},{"cell_type":"code","source":"\"\"\"\nSWA BatchNorm updating\n\"\"\"\nif CFG.use_swa:\n    since = time.time()\n    running_corrects = 0\n   \n    \n    \n    f1 = 0\n    f1_running = 0\n    \n    \n    #training\n    swa_model.train()\n    running_loss = 0.0\n    running_corrects = 0\n    iterations = int((CFG.trainnum + CFG.BATCH - 1) / CFG.BATCH)\n    m = 1\n    for i, (images, labels) in enumerate(trainloader):        \n        images = images.to(CFG.DEVICE, dtype=torch.int64)\n        labels = labels.to(CFG.DEVICE, dtype=torch.int64)\n\n        print(f\"Upgrading {m}/{iterations}\", end=\"\\r\")   \n        m += 1\n\n        with torch.set_grad_enabled(False) and autocast:\n            outputs = swa_model(images.float()) \n            loss, probs = criterion(outputs, labels)\n            f1_running += float(metric(outputs, labels))\n\n\n\n\n        # statistics\n        running_loss += loss.item() * images.size(0)\n        running_corrects = predictions(running_corrects, probs)\n\n  \n    f1_SWA= f1_running / iterations\n    epoch_loss = running_loss / len(trainset)\n    epoch_acc = running_corrects / len(trainset)\n    time_elapsed = time.time() - since\n    print('Loss: {:.6f} Acc: {:.4f} F1:{:.4f}'.format(\n    epoch_loss, epoch_acc, f1_SWA))\n    print('Updating BN of SWA model completed in {:.0f}m {:.0f}s'.format(\n    time_elapsed // 60, time_elapsed % 60)) \n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:48.209733Z","iopub.execute_input":"2021-06-03T15:02:48.210206Z","iopub.status.idle":"2021-06-03T15:02:48.225514Z","shell.execute_reply.started":"2021-06-03T15:02:48.210159Z","shell.execute_reply":"2021-06-03T15:02:48.224542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nSave SWA model\n\"\"\"\nif CFG.use_swa:\n    torch.save(swa_model.state_dict(),os.path.join(CFG.SWADIR))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:48.229432Z","iopub.execute_input":"2021-06-03T15:02:48.22985Z","iopub.status.idle":"2021-06-03T15:02:48.241445Z","shell.execute_reply.started":"2021-06-03T15:02:48.229787Z","shell.execute_reply":"2021-06-03T15:02:48.240443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nSWA validation\n\"\"\"\nif CFG.use_swa:\n    since = time.time()\n    running_corrects = 0\n\n    f1 = 0\n    f1_running = 0\n    \n    \n    #training\n    swa_model.eval()\n    running_loss = 0.0\n    running_corrects = 0\n    iterations = int((CFG.valnum + CFG.BATCH - 1) / CFG.BATCH)\n    m = 1\n    for i, (images, labels) in enumerate(valloader):        \n        images = images.to(CFG.DEVICE, dtype=torch.int64)\n        labels = labels.to(CFG.DEVICE, dtype=torch.int64)\n\n        print(f\"Upgrading {m}/{iterations}\", end=\"\\r\")   \n        m += 1\n\n        with torch.set_grad_enabled(False) and autocast:\n            outputs = swa_model(images.float()) \n            loss, probs = criterion(outputs, labels)\n            f1_running += float(metric(outputs, labels))\n\n        # statistics\n        running_loss += loss.item() * images.size(0)\n        running_corrects = predictions(running_corrects, probs)\n\n\n    f1_SWA= f1_running / iterations\n    epoch_loss = running_loss / len(valset)\n    epoch_acc = running_corrects / len(valset)\n    time_elapsed = time.time() - since\n    print('Loss: {:.6f} Acc: {:.4f} F1:{:.4f}'.format(\n    epoch_loss, epoch_acc, f1_SWA))\n    print('Validation of SWA model completed in {:.0f}m {:.0f}s'.format(\n    time_elapsed // 60, time_elapsed % 60)) ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:02:48.25892Z","iopub.execute_input":"2021-06-03T15:02:48.25981Z","iopub.status.idle":"2021-06-03T15:02:48.273177Z","shell.execute_reply.started":"2021-06-03T15:02:48.259729Z","shell.execute_reply":"2021-06-03T15:02:48.272177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Threshold tuning","metadata":{}},{"cell_type":"markdown","source":"In this chapter i minimized mean quantity of weighted mistakes over classes, where weighted mistake equals to amount of mistakes multiplied  by weight of this class. ","metadata":{}},{"cell_type":"code","source":"\"\"\"\nRedifine prediction function\n\"\"\"\ndef predictions(running_corrects, probs,count,mistakes):\n    predictions = probs > CFG.threshold\n    for batch in range(len(predictions)):\n        ill = 0\n        if torch.sum(predictions[batch] == labels.data[batch]) == CFG.num_classes:\n            running_corrects += 1\n        else:\n            for i in range(CFG.num_classes): \n                if labels.data[batch][i] == 1 and float(probs[batch][i]) < CFG.threshold[i] : \n                    count[i] += 1\n                    mistakes[i]=torch.add(mistakes[i], probs[batch][i])\n                    ill = 1\n                    \n            if ill == 0:\n                count[5] += 1\n                         \n    return running_corrects, count, mistakes","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:20:50.359814Z","iopub.execute_input":"2021-06-03T15:20:50.360194Z","iopub.status.idle":"2021-06-03T15:20:50.3691Z","shell.execute_reply.started":"2021-06-03T15:20:50.360146Z","shell.execute_reply":"2021-06-03T15:20:50.367106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nChoosing right threshold\n\"\"\"\nif CFG.THRESHOLD:\n    STEP = 0.005\n    ITERATIONS = 25\n\n    since = time.time()\n    a_weightedbest=torch.tensor(100000, device=CFG.DEVICE)\n    for num_cl in range(CFG.num_classes):\n\n        best_threshold = 0.5\n\n        for iterations in range(ITERATIONS):\n            #cycle begins with changed threshold in order to not compute twice with threshold = 0.5\n            CFG.threshold[num_cl] -= STEP\n            metric = MyF1Score(CFG)\n            print(\" \")\n            print(\"-\"*10)\n            print(\"Iteration: \",iterations, \"for class: \", num_cl)\n            print('Threshold = ', CFG.threshold)\n\n            running_loss = 0.0\n            running_corrects = 0\n            f1_running = 0\n\n            count=torch.zeros(CFG.num_classes + 1, device=CFG.DEVICE )\n            mistakes = torch.zeros(CFG.num_classes + 1, device=CFG.DEVICE )\n\n\n            model.eval()\n\n            iterations = int((CFG.valnum + CFG.BATCH - 1) / CFG.BATCH)\n            m = 1\n\n            for i, (images, labels) in enumerate(valloader):        \n                images = images.to(CFG.DEVICE, dtype=torch.int64)\n                labels = labels.to(CFG.DEVICE, dtype=torch.int64)       \n                optimizer.zero_grad()\n\n                print(f\"Training {m}/{iterations}\", end=\"\\r\")   \n                m += 1\n\n                # Runs the forward pass with autocasting.\n                with torch.set_grad_enabled(False) and autocast:\n                    outputs = model(images.float())\n                    loss, probs = criterion(outputs, labels)\n                    f1_running += float(metric(outputs, labels))     \n\n                # statistics\n                f1= f1_running / iterations\n                running_loss += loss.item() * images.size(0)\n                running_corrects, count, mistakes = predictions(running_corrects, probs,count,mistakes)\n\n\n\n            count_weighted = count * CFG.class_weights # vector of weighted amount of mistakes\n            print('count_weighted:', count_weighted)\n            print('count_weighted.mean: ', count_weighted.mean())   \n\n            #print('mistakes: ', mistakes/a) # Shows mean prediction of wrongly rejected class\n            #print('count:', count) #Vector with each element as amount of mistakes on particular class(last class is healthy)\n\n            if count_weighted.mean() <= count_weightedbest:\n                count_weightedbest = count_weighted.mean()\n                best_threshold = float(CFG.threshold[num_cl])\n                f1best = f1\n                print(\"Best threshold added with weighted mean mistakes: \", float(count_weightedbest))\n\n            epoch_loss = running_loss / len(valset)\n            epoch_acc = running_corrects / len(valset)\n            time_elapsed = time.time() - since\n            print('Validation Loss: {:.6f} Acc: {:.4f} F1:{:.4f}'.format(\n            epoch_loss, epoch_acc, f1))\n            print('Validation of threshold completed in {:.0f}m {:.0f}s'.format(\n            time_elapsed // 60, time_elapsed % 60)) \n        CFG.threshold[num_cl] = best_threshold\n        print(\"Best threshold for class{} is:{:.3f} with a_weighted.mean: {}\".format(num_cl,best_threshold,float(count_weightedbest)))\n\n    print(\"Best weighted threshold is: \", CFG.threshold) \n    print(\"F1: \", f1best)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:20:52.736706Z","iopub.execute_input":"2021-06-03T15:20:52.737125Z","iopub.status.idle":"2021-06-03T15:20:56.991017Z","shell.execute_reply.started":"2021-06-03T15:20:52.737078Z","shell.execute_reply":"2021-06-03T15:20:56.987671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make a prediction\n","metadata":{}},{"cell_type":"markdown","source":"On inference i used Images larger on 25% than on training.","metadata":{}},{"cell_type":"code","source":"Transformtest = transforms.Compose(\n    [transforms.ToTensor(),\n    transforms.Resize((CFG.IM_SIZE,CFG.IM_SIZE)),\n    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])","metadata":{"execution":{"iopub.status.busy":"2021-06-03T16:06:21.016099Z","iopub.execute_input":"2021-06-03T16:06:21.016527Z","iopub.status.idle":"2021-06-03T16:06:21.02517Z","shell.execute_reply.started":"2021-06-03T16:06:21.016497Z","shell.execute_reply":"2021-06-03T16:06:21.023843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_Test = [name for name in (os.listdir(CFG.TEST_DIR))] ","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:54:18.548071Z","iopub.execute_input":"2021-06-03T15:54:18.548523Z","iopub.status.idle":"2021-06-03T15:54:18.556397Z","shell.execute_reply.started":"2021-06-03T15:54:18.54849Z","shell.execute_reply":"2021-06-03T15:54:18.554886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testset = GetData(CFG.TEST_DIR, X_Test, None, Transformtest)\ntestloader = DataLoader(testset, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T16:06:24.555384Z","iopub.execute_input":"2021-06-03T16:06:24.555796Z","iopub.status.idle":"2021-06-03T16:06:24.562601Z","shell.execute_reply.started":"2021-06-03T16:06:24.555752Z","shell.execute_reply":"2021-06-03T16:06:24.561057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_num_to_str(pred: np.ndarray) -> str:\n    \"\"\"convert the numerical labels to string labels\"\"\"\n    lb_str_list = []\n    for lb_idx, bool_val in enumerate(pred):\n        if bool_val:\n            lb_str = CFG.label_num2str[lb_idx]\n            lb_str_list.append(lb_str)\n    if len(lb_str_list) == 0:\n        final_label = 'healthy'\n    else:\n        final_label = ' '.join(lb_str_list)\n    return final_label","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:54:22.520463Z","iopub.execute_input":"2021-06-03T15:54:22.520836Z","iopub.status.idle":"2021-06-03T15:54:22.52777Z","shell.execute_reply.started":"2021-06-03T15:54:22.520805Z","shell.execute_reply":"2021-06-03T15:54:22.526544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.DataFrame(columns=['image', 'labels'])\nsubmit_df['image'] = X_Test\nsubmit_df","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:54:24.15611Z","iopub.execute_input":"2021-06-03T15:54:24.156476Z","iopub.status.idle":"2021-06-03T15:54:24.170058Z","shell.execute_reply.started":"2021-06-03T15:54:24.156445Z","shell.execute_reply":"2021-06-03T15:54:24.168711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nUse SWA if it's available\n\"\"\"\nif CFG.use_swa and f1_SWA > f1:\n    model = swa_model","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:54:03.42445Z","iopub.execute_input":"2021-06-03T15:54:03.424874Z","iopub.status.idle":"2021-06-03T15:54:03.433108Z","shell.execute_reply.started":"2021-06-03T15:54:03.424841Z","shell.execute_reply":"2021-06-03T15:54:03.428905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\n\nwith torch.no_grad():\n    test_img_idx = 0\n    for img_ts, lb_ts in testloader:\n        img_ts = img_ts.cuda()\n        n_fold_pred_list = [] \n        pred_ts = torch.sigmoid(model(img_ts)).detach().cuda()\n        n_fold_pred_list.append(pred_ts)\n        pred_np = torch.cat(n_fold_pred_list).mean(dim=0)\n        print(pred_np)\n        pred = (pred_np > CFG.threshold).tolist()\n        \n        # convert numerical label into string\n        final_label = convert_num_to_str(pred)\n        img_name = lb_ts[test_img_idx]\n        row_idx = submit_df[submit_df.image == img_name].index.tolist()[0]\n        submit_df.iloc[row_idx, 1] = final_label","metadata":{"execution":{"iopub.status.busy":"2021-06-03T16:06:27.783436Z","iopub.execute_input":"2021-06-03T16:06:27.783834Z","iopub.status.idle":"2021-06-03T16:06:29.565293Z","shell.execute_reply.started":"2021-06-03T16:06:27.783802Z","shell.execute_reply":"2021-06-03T16:06:29.563799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nSave prediction\n\"\"\"\nsubmit_df.to_csv(\"./submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:55:33.85638Z","iopub.execute_input":"2021-06-03T15:55:33.856754Z","iopub.status.idle":"2021-06-03T15:55:33.862817Z","shell.execute_reply.started":"2021-06-03T15:55:33.856709Z","shell.execute_reply":"2021-06-03T15:55:33.861302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df","metadata":{"execution":{"iopub.status.busy":"2021-06-03T15:55:35.216596Z","iopub.execute_input":"2021-06-03T15:55:35.217022Z","iopub.status.idle":"2021-06-03T15:55:35.228287Z","shell.execute_reply.started":"2021-06-03T15:55:35.216989Z","shell.execute_reply":"2021-06-03T15:55:35.226991Z"},"trusted":true},"execution_count":null,"outputs":[]}]}