{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport PIL\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport random\nfrom tqdm import tqdm\nimport tensorflow_addons as tfa\nimport random\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport torch\nimport torchvision\nimport torchvision.models as models\nfrom PIL import Image\nimport torchvision.transforms as transforms\nimport os\nfrom torchvision import datasets\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport tqdm\nfrom tqdm import tqdm\nimport torch.optim as optim\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\nfrom torch.utils.data import Dataset, DataLoader\nimport cv2                \nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score\nfrom albumentations import *\nimport albumentations\nfrom albumentations.pytorch.transforms import ToTensorV2\nimport sys","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-16T05:26:48.791082Z","iopub.execute_input":"2022-09-16T05:26:48.792028Z","iopub.status.idle":"2022-09-16T05:26:59.139594Z","shell.execute_reply.started":"2022-09-16T05:26:48.791919Z","shell.execute_reply":"2022-09-16T05:26:59.138608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install --upgrade efficientnet-pytorch","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:26:59.141540Z","iopub.execute_input":"2022-09-16T05:26:59.142189Z","iopub.status.idle":"2022-09-16T05:27:11.359455Z","shell.execute_reply.started":"2022-09-16T05:26:59.142154Z","shell.execute_reply":"2022-09-16T05:27:11.358051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.361183Z","iopub.execute_input":"2022-09-16T05:27:11.361861Z","iopub.status.idle":"2022-09-16T05:27:11.374578Z","shell.execute_reply.started":"2022-09-16T05:27:11.361821Z","shell.execute_reply":"2022-09-16T05:27:11.373583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ndf['labels'] = df['labels'].apply(lambda string: string.split(' '))\n#df=df[:50]\ns = list(df['labels'])\nmlb = MultiLabelBinarizer()\ntrainx = pd.DataFrame(mlb.fit_transform(s), columns=mlb.classes_, index=df.index)\n#df['image'] = df['image'].str.replace(r'.jpg', '')\ntrainx.insert(0, \"image\", df['image'], True)\ntrainx","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.377675Z","iopub.execute_input":"2022-09-16T05:27:11.378393Z","iopub.status.idle":"2022-09-16T05:27:11.463586Z","shell.execute_reply.started":"2022-09-16T05:27:11.378358Z","shell.execute_reply":"2022-09-16T05:27:11.462658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_df=pd.read_csv('../input/plant-pathology-2021-fgvc8/sample_submission.csv')\ntest_df=t_df.drop(['labels'], axis=1)\n#test_df['image'] = test_df['image'].str.replace(r'.jpg', '')\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.465817Z","iopub.execute_input":"2022-09-16T05:27:11.466192Z","iopub.status.idle":"2022-09-16T05:27:11.483912Z","shell.execute_reply.started":"2022-09-16T05:27:11.466157Z","shell.execute_reply":"2022-09-16T05:27:11.482887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df=trainx\ntrain_df.reset_index(drop=True,inplace=True)\ntest_df.reset_index(drop=True,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.485402Z","iopub.execute_input":"2022-09-16T05:27:11.485976Z","iopub.status.idle":"2022-09-16T05:27:11.491407Z","shell.execute_reply.started":"2022-09-16T05:27:11.485940Z","shell.execute_reply":"2022-09-16T05:27:11.490400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self,df,root_dir,transform=None,iftest=False):\n        self.df=df\n        self.root_dir=root_dir\n        self.transform=transform\n        self.iftest=iftest\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self,idx):\n        if torch.is_tensor(idx):\n            idx=idx.tolist()\n        img_name=self.root_dir+self.df.iloc[idx,0]\n#         print(img_name)\n        image= cv2.imread(img_name,cv2.IMREAD_COLOR)\n#         image= cv2.imread(img_name)\n#         print(img_name,image)\n        image= cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n#         image = Image.fromarray(image)\n#         print(type(image))\n        if self.transform:\n            image=self.transform(image=image)['image']\n        if self.iftest:\n            return image\n        labels=torch.tensor(np.argmax(self.df.iloc[idx,1:].values))\n#         labels=np.asarray(labels)\n#         labels=torch.from_numpy(labels.astype(np.int32))\n#         labels=labels.unsqueeze(-1)\n#         print(labels.shape)\n#         sample={'image':image,'labels':labels}\n        return (image,labels)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.493301Z","iopub.execute_input":"2022-09-16T05:27:11.494255Z","iopub.status.idle":"2022-09-16T05:27:11.505996Z","shell.execute_reply.started":"2022-09-16T05:27:11.494220Z","shell.execute_reply":"2022-09-16T05:27:11.504747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE=545\nIMSIZE=EfficientNet.get_image_size('efficientnet-b5')\nprint(IMSIZE)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.507753Z","iopub.execute_input":"2022-09-16T05:27:11.508518Z","iopub.status.idle":"2022-09-16T05:27:11.521258Z","shell.execute_reply.started":"2022-09-16T05:27:11.508482Z","shell.execute_reply":"2022-09-16T05:27:11.519934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset=CustomDataset(df=train_df,root_dir='../input/plant-pathology-2021-fgvc8/train_images/',\n                     transform=Compose([augmentations.geometric.resize.Resize(height=IMSIZE,width=IMSIZE,always_apply=True),\n                                                  HorizontalFlip(p=0.5),\n                                                  VerticalFlip(p=0.5),\n                                                  ShiftScaleRotate(rotate_limit=25.0,p=0.7),\n                                                  OneOf([Emboss(p=1),Sharpen(p=1),Blur(p=1)],p=0.5),\n                                                  PiecewiseAffine(p=0.5),\n                                                   Normalize((0.485,0.456,0.406),\n                                                                      (0.229,0.224,0.225),always_apply=True),\n                                                  ToTensorV2()\n                                                  ]))","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.523366Z","iopub.execute_input":"2022-09-16T05:27:11.524059Z","iopub.status.idle":"2022-09-16T05:27:11.534288Z","shell.execute_reply.started":"2022-09-16T05:27:11.523966Z","shell.execute_reply":"2022-09-16T05:27:11.533359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_dataset=CustomDataset(df=train_df,root_dir='../input/plant-pathology-2021-fgvc8/train_images/',\n                     transform=Compose([augmentations.geometric.resize.Resize(height=IMSIZE,width=IMSIZE,always_apply=True),\n                                                  HorizontalFlip(p=0.5),\n                                                  VerticalFlip(p=0.5),\n                                                  ShiftScaleRotate(rotate_limit=25.0,p=0.7),\n                                                  OneOf([Emboss(p=1),Sharpen(p=1),Blur(p=1)],p=0.5),\n                                                  PiecewiseAffine(p=0.5),\n                                                   Normalize((0.485,0.456,0.406),\n                                                                      (0.229,0.224,0.225),always_apply=True),\n                                                  ToTensorV2()\n                                                  ]),iftest=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.538519Z","iopub.execute_input":"2022-09-16T05:27:11.539049Z","iopub.status.idle":"2022-09-16T05:27:11.548588Z","shell.execute_reply.started":"2022-09-16T05:27:11.539023Z","shell.execute_reply":"2022-09-16T05:27:11.547613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset=CustomDataset(df=test_df,root_dir='../input/plant-pathology-2021-fgvc8/test_images/',\n                     transform=Compose([augmentations.geometric.resize.Resize(height=IMSIZE,width=IMSIZE,always_apply=True),\n                                                  Normalize((0.485,0.456,0.406),\n                                                                      (0.229,0.224,0.225),always_apply=True),\n                                                    ToTensorV2()\n                                                  ]),iftest=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.549910Z","iopub.execute_input":"2022-09-16T05:27:11.550805Z","iopub.status.idle":"2022-09-16T05:27:11.558151Z","shell.execute_reply.started":"2022-09-16T05:27:11.550771Z","shell.execute_reply":"2022-09-16T05:27:11.557171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=1\ntrain_loader=DataLoader(train_dataset,batch_size=BATCH_SIZE,shuffle=True,num_workers=2)\nt_loader=DataLoader(t_dataset,batch_size=BATCH_SIZE,shuffle=False,num_workers=2)\ntest_loader=DataLoader(test_dataset,batch_size=BATCH_SIZE,shuffle=False,num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.561465Z","iopub.execute_input":"2022-09-16T05:27:11.562390Z","iopub.status.idle":"2022-09-16T05:27:11.571284Z","shell.execute_reply.started":"2022-09-16T05:27:11.562358Z","shell.execute_reply":"2022-09-16T05:27:11.570326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_cuda = torch.cuda.is_available()\nif use_cuda:\n    device='cuda:0'\nuse_tpu=False\nuse_device=True\nif use_tpu:\n    device='idk'","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.573071Z","iopub.execute_input":"2022-09-16T05:27:11.573839Z","iopub.status.idle":"2022-09-16T05:27:11.643152Z","shell.execute_reply.started":"2022-09-16T05:27:11.573805Z","shell.execute_reply":"2022-09-16T05:27:11.642037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(n_epochs,train_loader,valid_loader,model,optimizer,criterion,use_device,save_path,final_train=False,ifsched=False):\n    for epoch in range(1,n_epochs+1):\n        train_loss=0.0\n        valid_loss=0.0\n        labels_for_acc=[]\n        output_for_acc=[]\n        labels_for_accv=[]\n        output_for_accv=[]\n        model.train()\n        for batch_idx,(data,target) in enumerate(tqdm(train_loader)):\n            #print(batch_idx)\n#            print(type(data),type(target))\n            if use_device:\n                data,target=data.to(device),target.to(device)\n            optimizer.zero_grad()\n            output=model(data)\n            loss=criterion(output,target)\n            train_loss+=loss.item()*data.size(0)\n            loss.backward()\n            optimizer.step()\n            if ifsched:\n                    scheduler.step()\n            labels_for_acc=np.concatenate((labels_for_acc,target.cpu().numpy()),0)\n            output_for_acc=np.concatenate((output_for_acc,np.argmax(output.cpu().detach().numpy(),1)),0)\n        train_loss=train_loss/len(train_loader.dataset)\n        train_acc=accuracy_score(labels_for_acc,output_for_acc)\n        if not final_train:\n            with torch.no_grad():\n                model.eval()\n                for batch_idx,(data,target) in enumerate(valid_loader):\n                    if use_device:\n                        data,target=data.to(device),target.to(device)\n                    output=model(data)\n                    loss=criterion(output,target)\n                    valid_loss+=loss.item()*data.size(0)\n                    labels_for_accv=np.concatenate((labels_for_accv,target.cpu().numpy()),0)\n                    output_for_accv=np.concatenate((output_for_accv,np.argmax(output.cpu().detach().numpy(),1)),0)\n                valid_loss=valid_loss/len(valid_loader.dataset)\n                valid_acc=accuracy_score(labels_for_accv,output_for_accv)\n                print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f} \\tTrain Acc: {:.6f} \\tValidation Acc: {:.6f}'.format(\n                epoch, \n                train_loss,\n                valid_loss,\n                train_acc,\n                valid_acc\n                ))\n                #checkpoint = ModelCheckpoint('model-{epoch:03d}-{acc:03f}-{val_acc:03f}.h5', verbose=1, monitor='val_loss',save_best_only=True, mode='auto')\n        if final_train:\n            print('Epoch: {} \\tTraining Loss: {:.6f} \\tTrain Acc: {:.6f} '.format(\n                epoch, \n                train_loss,\n                train_acc\n                ))\n            #checkpoint = ModelCheckpoint('model-{epoch:03d}-{acc:03f}-{val_acc:03f}.h5', verbose=1, monitor='val_loss',save_best_only=True, mode='auto')\n        #return train_loss, optimizer","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.644869Z","iopub.execute_input":"2022-09-16T05:27:11.645591Z","iopub.status.idle":"2022-09-16T05:27:11.660567Z","shell.execute_reply.started":"2022-09-16T05:27:11.645556Z","shell.execute_reply":"2022-09-16T05:27:11.659574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_efficient=EfficientNet.from_pretrained('efficientnet-b7')","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:11.662326Z","iopub.execute_input":"2022-09-16T05:27:11.663187Z","iopub.status.idle":"2022-09-16T05:27:15.569885Z","shell.execute_reply.started":"2022-09-16T05:27:11.663089Z","shell.execute_reply":"2022-09-16T05:27:15.568668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for param in model_efficient.parameters():\n#     param.requires_gr\nad=False\n# print(model_transfer)\nmodel_efficient._fc=nn.Sequential(nn.Linear(model_efficient._fc.in_features,1000,bias=True),\n                                 nn.ReLU(),\n                                 nn.Dropout(p=0.5),\n                                 nn.Linear(1000,6,bias=True))\n# nn.init.kaiming_normal_(model_efficient._fc.weight, nonlinearity='relu')\nif use_device:\n    model_efficient = model_efficient.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T05:27:15.571524Z","iopub.execute_input":"2022-09-16T05:27:15.572200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEPOCHS=4\nprint(IMSIZE)\ncriterion_transfer = nn.CrossEntropyLoss()\n# learning_rate=5e-4*np.logspace(0,1.5,9)\n# learning_rate=learning_rate[2]\nlearning_rate=8e-4\noptimizer_transfer = optim.AdamW(model_efficient.parameters(),learning_rate,weight_decay=1e-3)\nnum_train_steps = int(len(train_dataset) / BATCH_SIZE * NEPOCHS)\nfrom transformers import get_cosine_schedule_with_warmup\nscheduler = get_cosine_schedule_with_warmup(optimizer_transfer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps)\n# optimizer_transfer = torch.optim.Adam(model_efficient.parameters())\n# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer_transfer, 'max', patience = 3,verbose=True,min_lr=0.00001)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(NEPOCHS, train_loader,None, model_efficient, optimizer_transfer, criterion_transfer, use_device, 'model_transfer.pt',ifsched=True,final_train=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!touch 'weghits_TB1.pt'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.save(model_efficient.state_dict(),'/kaggle/working/weghits_1.h5')\nimport copy\nfrom copy import deepcopy\nbest_model_state = copy.deepcopy(model_efficient.state_dict())\ntorch.save(model_efficient.state_dict(), '/kaggle/working/weghits_TB1.pt' )","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}