{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Lib"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nimport json\nimport torch\nimport random\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\nsys.path.append('../input/repvgg')\nfrom repvgg import repvgg_model_convert, create_RepVGG_B1\nimport torch.nn as nn\nimport torchvision \nfrom torchvision import models,transforms\nfrom PIL import Image\nfrom torch.utils.data import Dataset , DataLoader \n\n%matplotlib inline\nBASE_DIR = \"../input/plant-pathology-2021-fgvc8\"\nBASE_TRAIN_IMAGES_DIR = \"../input/plant-pathology-2021-fgvc8/train_images\"\nBASE_TEST_IMAGES_DIR = \"../input/plant-pathology-2021-fgvc8/test_images\"\nDEVICE=torch.device(\"cuda\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_it(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\nseed_it(31)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.backends.cudnn.benchmark = True\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(BASE_DIR,'train.csv'))\ntrain_df.info()\ntrain_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encode Label\nreference:[this notebook](https://www.kaggle.com/ateplyuk/plant-2021-starter)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\n\nle = preprocessing.LabelEncoder()\nle.fit(train_df['labels'])\ntrain_df['labels_id'] = le.transform(train_df['labels'])\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASS_NAMES=dict(sorted(train_df[['labels_id', 'labels']].values.tolist()))\nCLASS_NAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,axis =plt.subplots() \nlabel_counts=train_df[\"labels\"].value_counts()\nlabel_counts_names=label_counts.index.tolist()\nlabel_counts=label_counts.values\n\naxis.barh(label_counts_names, label_counts, align='center')\n\naxis.invert_yaxis()\nfig.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check Image Size"},{"metadata":{"trusted":true},"cell_type":"code","source":"def checkimagesize(paths):\n    sizes={}\n    for p in paths:\n        img = Image.open(os.path.join(BASE_TRAIN_IMAGES_DIR,p))\n        if str(img.size) in sizes:\n            sizes[str(img.size)]+=1\n        else:\n            sizes[str(img.size)]=1\n    print(sizes)\n#checkimagesize(train_df['image'])\n'''\n{'(4000, 2672)': 16485,\n'(4000, 3000)': 665,\n'(2592, 1728)': 1027,\n'(4608, 3456)': 123,\n'(5184, 3456)': 193,\n'(4032, 3024)': 132,\n'(3024, 4032)': 3,\n'(3024, 3024)': 3,\n'(4000, 2248)': 1}\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class PlantDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        \n        super().__init__()\n        self.dataframe = df\n        self.transforms = transforms\n    \n    def __len__(self):\n        return self.dataframe.shape[0]\n    \n    def __getitem__(self, index: int):\n\n        label = self.dataframe.iloc[index]['labels_id']\n          \n        imgpath = os.path.join(BASE_TRAIN_IMAGES_DIR,self.dataframe.iloc[index][\"image\"])\n        img = Image.open(imgpath)\n        if self.transforms:\n            img = self.transforms(img)\n        return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def splitData(dataframe,p=0.8):\n    randomlist = np.random.rand(len(dataframe))<p\n    train_dataframe =  dataframe[randomlist]\n    valid_dataframe =dataframe[~randomlist]\n    print(\"train {}\".format(len(train_dataframe)))\n    print(\"valid {}\".format(len(valid_dataframe)))\n    return train_dataframe , valid_dataframe\n\ntrainDataframe,validDataframe=splitData(train_df)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Weighted Sampler for Balence Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_weight_for_balance(dataframe,numclass):\n    numdata = len(dataframe)\n    counts=[0]*numclass\n    for l in range(numclass):\n        counts[l]=len(dataframe[dataframe[\"labels_id\"]==l])\n    weights_per_classes=[0]*numclass\n    for idx,c in enumerate(counts):\n        weights_per_classes[idx] = 0 if c ==0 else (numdata/c)\n    print(weights_per_classes)\n    weights=[]\n    for i in range(numdata):\n        weights.append(weights_per_classes[dataframe.iloc[i][\"labels_id\"]])\n    return torch.DoubleTensor(weights)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"Weighted=False\n\nBATCH_SIZE=64\n\nHEIGHT,WIDTH=224,224\n\nnumclasses = len(CLASS_NAMES.values())\n\ntrain_transform = transforms.Compose([\n                                transforms.Resize((WIDTH,HEIGHT)),\n                                #transforms.RandomCrop(400,300),\n                                transforms.RandomHorizontalFlip(),\n                                transforms.RandomVerticalFlip(),\n                                #transforms.RandomRotation(90),\n                                #transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1, hue=0.1),\n                                transforms.ToTensor(),\n                                #transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))])\n                                transforms.Normalize((0.485, 0.456, 0.406),(.229, 0.224, 0.225))])\n\nvalid_transform = transforms.Compose([transforms.Resize((WIDTH,HEIGHT)),\n                                    transforms.ToTensor(),\n                                    transforms.Normalize((0.485, 0.456, 0.406),(.229, 0.224, 0.225))])\n\ntrainDataset = PlantDataset(trainDataframe,train_transform)\nvalidDataset = PlantDataset(validDataframe,valid_transform)\n\nif Weighted:\n    weights = get_weight_for_balance(trainDataframe,numclasses)\n    weightsampler = WeightedRandomSampler(torch.DoubleTensor(weights),num_samples=8000, replacement=True)\n    trainDataLoader = DataLoader(trainDataset,batch_size= BATCH_SIZE,num_workers=4,pin_memory=True,shuffle=False,sampler=weightsampler)\nelse:\n    trainDataLoader = DataLoader(trainDataset,batch_size= BATCH_SIZE,num_workers=4,pin_memory=True,shuffle=True)\n    \nvalidDataLoader = DataLoader(validDataset,batch_size= BATCH_SIZE,num_workers=4,pin_memory=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Model"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model = create_RepVGG_B1(deploy=False)\nmodel.load_state_dict(torch.load(\"../input/repvggpretrainedweights/drive-download-20210121T111115Z-001/RepVGG-B1-train.pth\"))\nin_features = model.linear.in_features\nmodel.linear = nn.Linear(in_features,numclasses,bias=True)\nprint(model)\nmodel=model.to(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Warmup LearningRateScheler"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / DECAY_EPOCHS) * math.pi\n        decay_factor= (math.cos(decay_factor) + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCH=20\nLR_START = 1e-6\nLR_MAX = 2e-4\nLR_FINAL = 1e-6\nLR_RAMPUP_EPOCHS = 2\nLR_SUSTAIN_EPOCHS = 0\nDECAY_EPOCHS = EPOCH  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\nLR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCH - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\ndef show_lr_schedule(epochs):\n    rng = [i for i in range(epochs)]\n    y = [lrfn(x) for x in rng]\n    x = np.arange(epochs)\n    x_axis_labels = list(map(str, np.arange(1, epochs+1)))\n    print('init lr {:.1e} to {:.1e} final {:.1e}'.format(y[0], max(y), y[-1]))\n    \n    plt.figure(figsize=(30, 10))\n    plt.xticks(x, x_axis_labels, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    plt.plot(rng, y)\n    plt.grid()\n    plt.show() \nshow_lr_schedule(EPOCH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def change_lr(op,epoch):\n    newlr = lrfn(epoch)\n    optimizer.param_groups[0]['lr'] = newlr*0.5\n    optimizer.param_groups[1]['lr'] = newlr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Loss function and Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntopparams=[]\nfor name,params in model.named_parameters():\n    if \"linear\" not in name:\n        topparams.append(params)\nparams=[{'params':topparams,'lr':LR_START*0.5},\n        {'params':model.linear.parameters(),'lr':LR_START},]\noptimizer = torch.optim.AdamW(params=params,lr=LR_START)\nlossfunction = torch.nn.CrossEntropyLoss()\n#scheduler=torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.25, patience=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define M1-Score Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calcaulateMacroF1(allpred,allans,allpredacc,nclasses):\n    recalls = [0 if allans[i] == 0 else round(allpredacc[i]/allans[i],2) for  i in range(0,5)]\n    precisions = [0 if allpred[i] == 0 else round(allpredacc[i]/allpred[i],2) for  i in range(0,5)]\n    avg_recalls = sum(recalls) / nclasses\n    avg_precisions = sum(precisions) / nclasses\n    macro_f1 = (0 if (avg_recalls+avg_precisions) == 0 else 2*(avg_recalls*avg_precisions)/(avg_recalls+avg_precisions))\n    recalls = [\"%.2f\"%i for i in recalls]\n    precisions = [\"%.2f\"%i for i in precisions]\n    return recalls ,precisions ,macro_f1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"minLoss = 1.0\n\ndef train_one_epoch(dataloader , model):\n    \n    model.train()\n    total_loss=0\n    iter_count=0\n    total_acc=[0]*numclasses\n    pred_acc=[0]*numclasses\n    allpred=[0]*numclasses\n    total_iter=len(dataloader)\n    \n    for imgs,labels in dataloader:\n        \n        iter_count+=1\n        imgs = imgs.to(DEVICE)\n        labels=labels.to(DEVICE)\n        \n        pred = model(imgs)\n        \n        loss = lossfunction(pred,labels)  \n        loss.backward()\n        total_loss+=loss.detach()\n        \n        for p_index , p in enumerate(pred):\n            p_label = p.argmax()\n            allpred[p_label]+=1\n            total_acc[labels[p_index]]+=1\n            if p_label == labels[p_index]:\n                pred_acc[p_label]+=1\n        \n        recalls ,precisions ,macro_f1 = calcaulateMacroF1(allpred,total_acc,pred_acc,numclasses)\n        avg_acc = [\"%.3f\"%(pred_acc[i]/(1 if total_acc[i]==0 else total_acc[i])) for i in range(numclasses)]\n        avg_loss = total_loss/iter_count           \n        print(\"\\rTrain {}/{} Loss:{} Acc:{} F1:{} Recall{} Precisions{}\".format(iter_count,total_iter,\"%.3f\"%avg_loss,\"%.3f\"%(sum(pred_acc)/sum(total_acc)),\"%.2f\"%macro_f1,recalls,precisions),end='',flush=True)\n    print('')\n    \n    \ndef evaluate(dataloader,model,epoch):\n    \n    global minLoss\n    model.eval()\n    total_loss=0\n    total_acc=[0]*numclasses\n    pred_acc=[0]*numclasses\n    allpred=[0]*numclasses\n    iter_count=0\n    total_iter=len(dataloader)\n    \n    with torch.no_grad():\n        for imgs,labels in dataloader:\n            \n            iter_count+=1\n            imgs = imgs.to(DEVICE)\n            labels=labels.to(DEVICE)\n            \n            pred = model(imgs)\n            loss = lossfunction(pred,labels)\n            total_loss+=loss.detach()\n            \n            for p_index , p in enumerate(pred):\n                p_label = p.argmax()\n                allpred[p_label]+=1\n                total_acc[labels[p_index]]+=1\n                if p_label == labels[p_index]:\n                    pred_acc[p_label]+=1\n                    \n            recalls ,precisions ,macro_f1 = calcaulateMacroF1(allpred,total_acc,pred_acc,numclasses)\n            avg_acc = [\"%.3f\"%(pred_acc[i]/(1 if total_acc[i]==0 else total_acc[i])) for i in range(numclasses)]\n            avg_loss = total_loss/iter_count\n            print(\"\\rValid {}/{} Loss:{} Acc:{} F1:{} Recall{} Precisions{}\".format(iter_count,total_iter,\"%.3f\"%avg_loss,\"%.3f\"%(sum(pred_acc)/sum(total_acc)),\"%.2f\"%macro_f1,recalls,precisions),end='',flush=True)\n            \n        #scheduler.step(avg_loss)\n        if avg_loss < minLoss:\n            minLoss = avg_loss\n            savemodel(model,f\"modelweight_{avg_loss}_{epoch}.pkl\")\n            \n    print('')\n    \ndef savemodel(model,filepath):\n    model_dir=\"/kaggle/working\"\n    torch.save(model.state_dict(),os.path.join(model_dir,filepath))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(0,EPOCH):\n    change_lr(optimizer,epoch)\n    print(\"EPOCH:{}\".format(epoch+1))\n    train_one_epoch(trainDataLoader,model)\n    evaluate(validDataLoader,model,epoch)\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}