{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport sklearn as sk\nimport cv2 \nfrom skimage.transform import resize\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.datasets as dataset\nimport torchvision.transforms as transforms\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport pandas as pd\nimport os\nimport gc\nimport copy\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport matplotlib.pyplot as plt\n\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch import nn\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nfrom pytorch_lightning import metrics\nfrom pytorch_lightning import LightningDataModule, LightningModule, Trainer\nfrom pytorch_lightning.callbacks import LearningRateMonitor\n\nfrom transformers import AdamW, get_cosine_schedule_with_warmup\n\nfrom tqdm.notebook import tqdm\n\nfrom PIL import Image\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-17T18:56:24.349430Z","iopub.execute_input":"2021-07-17T18:56:24.349734Z","iopub.status.idle":"2021-07-17T18:56:33.016268Z","shell.execute_reply.started":"2021-07-17T18:56:24.349672Z","shell.execute_reply":"2021-07-17T18:56:33.015363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN=False\nNUMBER_EPOCHS=8\nSUBMIT=True\nLOAD=True\nEVALUATE_TRESHOLD=True\nLOAD_CHECK_PATH='../input/train-checkpoints-effb5/checkpoints/efficientb5/epoch2.pb'","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.019560Z","iopub.execute_input":"2021-07-17T18:56:33.019829Z","iopub.status.idle":"2021-07-17T18:56:33.025112Z","shell.execute_reply.started":"2021-07-17T18:56:33.019804Z","shell.execute_reply":"2021-07-17T18:56:33.024082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## get train csv","metadata":{}},{"cell_type":"code","source":"train_csv='/kaggle/input/plant-pathology-2021-fgvc8/train.csv'\ntrain_df=pd.read_csv(train_csv)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.027075Z","iopub.execute_input":"2021-07-17T18:56:33.027398Z","iopub.status.idle":"2021-07-17T18:56:33.083603Z","shell.execute_reply.started":"2021-07-17T18:56:33.027363Z","shell.execute_reply":"2021-07-17T18:56:33.082674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Possible label Extraction","metadata":{}},{"cell_type":"code","source":"labels=train_df['labels'].to_numpy()\ncount_labels={}\nfor label in labels:\n  for word in label.split(' '):\n    if word not in count_labels:\n      count_labels[word]=1\n    else:\n      count_labels[word]+=1\npossible_labels=count_labels.keys()\nmap_dictionary={ label:index for index,label in enumerate(possible_labels)}\nmap_dictionary.keys()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.085212Z","iopub.execute_input":"2021-07-17T18:56:33.085563Z","iopub.status.idle":"2021-07-17T18:56:33.108546Z","shell.execute_reply.started":"2021-07-17T18:56:33.085529Z","shell.execute_reply":"2021-07-17T18:56:33.107796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## label to tensor casting","metadata":{}},{"cell_type":"code","source":"def get_label_from_image_name(img_name:str)->str:\n  return train_df.loc[train_df['image'] == img_name].labels.to_numpy()[0]\n\ndef label_to_tensor(label : str):\n  listed_classes=label.split(' ')\n  torch_target=torch.zeros([6], dtype=torch.float)\n  for class_found in listed_classes:\n    index=map_dictionary[class_found]\n    torch_target[index]=1\n  return torch_target\n\nst='scab frog_eye_leaf_spot complex'\nlabel_to_tensor(st)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.110026Z","iopub.execute_input":"2021-07-17T18:56:33.110248Z","iopub.status.idle":"2021-07-17T18:56:33.176358Z","shell.execute_reply.started":"2021-07-17T18:56:33.110226Z","shell.execute_reply":"2021-07-17T18:56:33.175621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Sources","metadata":{}},{"cell_type":"code","source":"base_image_dir='../input/resized-plant2021/img_sz_512'\nbase_test_dir='/kaggle/input/plant-pathology-2021-fgvc8/test_images'","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.178795Z","iopub.execute_input":"2021-07-17T18:56:33.179050Z","iopub.status.idle":"2021-07-17T18:56:33.184512Z","shell.execute_reply.started":"2021-07-17T18:56:33.179025Z","shell.execute_reply":"2021-07-17T18:56:33.183677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configurações\n   ","metadata":{}},{"cell_type":"code","source":"class CONFIG:\n    seed = 42\n    model_name = 'tf_efficientnetv2_m_in21k' \n    train_batch_size = 8\n    valid_batch_size = 32\n    img_size = (256,256)\n    epochs = 5\n    learning_rate = 1e-4\n    min_lr = 1e-6\n    weight_decay = 1e-6\n    T_max = 5\n    scheduler = None\n    n_accumulate = 1\n    n_fold = 5\n    target_size = 6\n    if (torch.cuda.is_available()):\n      print('using gpu')\n      device = torch.device(\"cuda\")\n    else:\n      print('using cpu')\n      device = torch.device(\"cpu\")\n\ndef set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG.seed)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.187420Z","iopub.execute_input":"2021-07-17T18:56:33.187711Z","iopub.status.idle":"2021-07-17T18:56:33.238784Z","shell.execute_reply.started":"2021-07-17T18:56:33.187686Z","shell.execute_reply":"2021-07-17T18:56:33.237675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataclasses for train and test","metadata":{}},{"cell_type":"code","source":"class PlantDataset(Dataset):\n    \n    def __init__(self,base_dir, paths ,transform=None ):\n        super().__init__()\n        \n        if transform is None:\n            self.transform=A.Compose([\n                    A.Resize(428, 428),\n                    ToTensorV2()\n            ])\n        else:\n            self.transform=transform\n            \n        self.base_dir=base_dir\n        self.paths=paths\n    \n        \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, idx):\n        img_loc = os.path.join(self.base_dir, self.paths[idx])\n        image = Image.open(img_loc).convert(\"RGB\")\n        image=np.array(image)\n        trans_image = self.transform(image=image)['image']\n        label=get_label_from_image_name(self.paths[idx])\n        label_tensor=label_to_tensor(label)\n        return (trans_image,label_tensor)\n    \n    \nclass PlantDatasetTest(Dataset):\n    \n    def __init__(self,base_dir, paths ,transform=None ):\n        super().__init__()\n        \n        if transform is None:\n            self.transform=A.Compose([\n                    A.Resize(428, 428),\n                    ToTensorV2()\n            ])\n        else:\n            self.transform=transform\n            \n        self.base_dir=base_dir\n        self.paths=paths\n    \n        \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, idx):\n        img_loc = os.path.join(self.base_dir, self.paths[idx])\n        image = Image.open(img_loc).convert(\"RGB\")\n        image=np.array(image)\n        trans_image = self.transform(image=image)['image']\n        label=self.paths[idx]\n        return (trans_image,label)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.242337Z","iopub.execute_input":"2021-07-17T18:56:33.242585Z","iopub.status.idle":"2021-07-17T18:56:33.253735Z","shell.execute_reply.started":"2021-07-17T18:56:33.242560Z","shell.execute_reply":"2021-07-17T18:56:33.252821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Dataloader for train and validation","metadata":{}},{"cell_type":"code","source":"paths=os.listdir(base_image_dir)\ntrain_paths=paths[:-1840]\nvalidation_paths=paths[-1840:]\ntransforms=A.Compose([\n                A.Resize(428, 428),\n                ToTensorV2()\n        ])\n\ntrain_loader = DataLoader(PlantDataset(base_image_dir,train_paths,transforms), shuffle=False,num_workers=2,batch_size=CONFIG.train_batch_size)\nvalidation_loader = DataLoader(PlantDataset(base_image_dir,validation_paths,transforms), shuffle=False,num_workers=2,batch_size=CONFIG.train_batch_size)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.256037Z","iopub.execute_input":"2021-07-17T18:56:33.256544Z","iopub.status.idle":"2021-07-17T18:56:33.776744Z","shell.execute_reply.started":"2021-07-17T18:56:33.256510Z","shell.execute_reply":"2021-07-17T18:56:33.775894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating preprocess class","metadata":{}},{"cell_type":"code","source":"from torch import nn\nfrom torchvision import transforms as T\n\nclass Preprocess(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        \n        self.augment = nn.Sequential(\n            T.RandomCrop((400, 400)),\n            T.RandomApply(nn.ModuleList([T.CenterCrop((385,385))]),p=.5),\n            T.RandomHorizontalFlip(.5),\n            T.RandomVerticalFlip(.5),\n            T.ColorJitter(brightness=.1, hue=.1,contrast=.1)\n        )\n        \n        self.normalize = nn.Sequential(\n            T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        )\n        \n    def forward(self,x):\n        x = x / 255 # move to 0,1 interval \n        x = self.augment(x) # apply augment\n        return self.normalize(x)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.778145Z","iopub.execute_input":"2021-07-17T18:56:33.778573Z","iopub.status.idle":"2021-07-17T18:56:33.788376Z","shell.execute_reply.started":"2021-07-17T18:56:33.778535Z","shell.execute_reply":"2021-07-17T18:56:33.787521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install ../input/efficientnet-pytorch-offline-pack/EfficientNet-PyTorch\n","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:56:33.789446Z","iopub.execute_input":"2021-07-17T18:56:33.789721Z","iopub.status.idle":"2021-07-17T18:57:02.550954Z","shell.execute_reply.started":"2021-07-17T18:56:33.789691Z","shell.execute_reply":"2021-07-17T18:57:02.549793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Efficientnet Model","metadata":{}},{"cell_type":"code","source":"\nfrom efficientnet_pytorch import EfficientNet\n\nclass Model(LightningModule):\n    \n    def __init__(self, lr=5e-3,betas=(.8, .9),weight_decay=10.,drop1=.3,drop2=.1):\n        super().__init__()\n        \n        self.save_hyperparameters()\n        \n        self.cost = nn.BCELoss()\n        \n        self.preprocess = Preprocess()\n        \n        def load_effnet(i,path,out):\n            effnet = EfficientNet.from_name('efficientnet-b'+str(i))\n            effnet.load_state_dict(torch.load(path))\n            n_features = effnet._fc.in_features\n            effnet._dropout = nn.Dropout(self.hparams.drop1)\n            effnet._fc = nn.Sequential(\n                            nn.Linear(n_features,1000),nn.ReLU(),\n                            nn.Dropout(self.hparams.drop2),\n                            nn.Linear(1000,out)\n            )\n            return effnet\n        \n        self.effnet = load_effnet(5,\n                                  '../input/efficientnet-pytorch-offline-pack/efficientnet-b5-b6417697.pth',6)\n        \n        \n    def forward(self,x):\n        x = self.preprocess(x)\n        x = self.effnet(x)\n        return torch.sigmoid(x)\n\n    \nmodel = Model()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:02.555317Z","iopub.execute_input":"2021-07-17T18:57:02.555578Z","iopub.status.idle":"2021-07-17T18:57:04.490768Z","shell.execute_reply.started":"2021-07-17T18:57:02.555551Z","shell.execute_reply":"2021-07-17T18:57:04.489946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# getting optimizer parameters","metadata":{}},{"cell_type":"code","source":"params_to_update = model.parameters()\nfeature_extract = True\nprint(\"Params to learn:\")\nif feature_extract:\n    params_to_update = []\n    for name,param in model.named_parameters():\n        if param.requires_grad == True:\n            params_to_update.append(param)\n            print(\"\\t\",name)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:04.492029Z","iopub.execute_input":"2021-07-17T18:57:04.492353Z","iopub.status.idle":"2021-07-17T18:57:04.618934Z","shell.execute_reply.started":"2021-07-17T18:57:04.492320Z","shell.execute_reply":"2021-07-17T18:57:04.606013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN:\n    optimizer = optim.Adam(params_to_update, lr=CONFIG.learning_rate, weight_decay=CONFIG.weight_decay)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:04.627651Z","iopub.execute_input":"2021-07-17T18:57:04.627891Z","iopub.status.idle":"2021-07-17T18:57:04.654386Z","shell.execute_reply.started":"2021-07-17T18:57:04.627867Z","shell.execute_reply":"2021-07-17T18:57:04.637281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## loading model to device","metadata":{}},{"cell_type":"code","source":"model.to(CONFIG.device)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:04.657947Z","iopub.execute_input":"2021-07-17T18:57:04.658222Z","iopub.status.idle":"2021-07-17T18:57:10.075989Z","shell.execute_reply.started":"2021-07-17T18:57:04.658194Z","shell.execute_reply":"2021-07-17T18:57:10.075019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### creating folders to store checkpoints","metadata":{}},{"cell_type":"code","source":"os.makedirs('./checkpoints/efficientb5/',exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.077484Z","iopub.execute_input":"2021-07-17T18:57:10.078052Z","iopub.status.idle":"2021-07-17T18:57:10.082666Z","shell.execute_reply.started":"2021-07-17T18:57:10.078011Z","shell.execute_reply":"2021-07-17T18:57:10.081927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training structure","metadata":{}},{"cell_type":"code","source":"import copy\n\ndef train_epoch(model,train_loader,optimizer,epoch):\n    model.train()\n    print('training epoch', epoch)\n    running_loss = 0.0\n    train_loss = 0.0\n    for i, data in enumerate(train_loader, 0):\n        # get the inputs; data is a list of [inputs, labels]\n        inputs, labels = data\n        inputs=inputs.to(CONFIG.device)\n        labels=labels.to(CONFIG.device)\n\n        # zero the parameter gradients\n        optimizer.zero_grad()\n\n        # forward + backward + optimize\n        outputs = model(inputs)\n#             print('\\n new\\n')\n#             print(outputs.shape, outputs)\n\n#             prob_outputs=softmax(outputs)\n\n        loss = model.cost(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        # print statistics\n        running_loss += loss.item() * CONFIG.train_batch_size\n        if i % 100 == 99:    # print every 100 mini-batches\n            print('[%d, %5d] loss: %.3f' %\n                  (epoch + 1, i + 1, running_loss / 100))\n            train_loss+=running_loss\n            running_loss = 0.0\n\n    train_loss = train_loss/len(train_loader.sampler)\n    \n    return train_loss","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.083983Z","iopub.execute_input":"2021-07-17T18:57:10.084457Z","iopub.status.idle":"2021-07-17T18:57:10.094016Z","shell.execute_reply.started":"2021-07-17T18:57:10.084417Z","shell.execute_reply":"2021-07-17T18:57:10.093233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef valid_epoch(model,validation_loader,optimizer,epoch):\n    model.eval()\n    print('validation epoch', epoch)\n    running_loss = 0.0\n    val_loss=0.0\n    for i, data in enumerate(validation_loader, 0):\n        # get the inputs; data is a list of [inputs, labels]\n        inputs, labels = data\n        inputs=inputs.to(CONFIG.device)\n        labels=labels.to(CONFIG.device)\n\n        \n        # forward + backward + optimize\n        outputs = model(inputs)\n#             print('\\n new\\n')\n#             print(outputs.shape, outputs)\n\n#             prob_outputs=softmax(outputs)\n\n        loss = model.cost(outputs, labels)\n\n        # print statistics\n        running_loss += loss.item() * CONFIG.train_batch_size\n        if i % 100 == 99:    # print every 2000 mini-batches\n            print('[%d, %5d] loss: %.3f' %\n                  (epoch + 1, i + 1, running_loss / 100))\n            val_loss+=running_loss\n            running_loss = 0.0\n    \n    val_loss = val_loss/len(validation_loader.sampler)\n    return val_loss","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.095168Z","iopub.execute_input":"2021-07-17T18:57:10.095651Z","iopub.status.idle":"2021-07-17T18:57:10.108213Z","shell.execute_reply.started":"2021-07-17T18:57:10.095614Z","shell.execute_reply":"2021-07-17T18:57:10.107056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport gc\ndef run_train_validation_epochs(model, optimizer,train_loader,valid_loader,num_epochs):    \n    start = time.time()\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        train_epoch_loss = train_epoch(model,train_loader=train_loader,optimizer=optimizer,epoch=epoch)\n        \n        valid_epoch_loss = valid_epoch(model, optimizer=optimizer,validation_loader=valid_loader,epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n        history['Valid Loss'].append(valid_epoch_loss)\n\n        print(f'loss in epoch {epoch} train={train_epoch_loss} valid={valid_epoch_loss}')\n\n        PATH = \"./checkpoints/efficientb5/epoch{:.0f}.pb\".format(epoch)\n        torch.save(model.state_dict(), PATH)\n        print(\"Model Saved\")    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.109454Z","iopub.execute_input":"2021-07-17T18:57:10.110007Z","iopub.status.idle":"2021-07-17T18:57:10.122642Z","shell.execute_reply.started":"2021-07-17T18:57:10.109967Z","shell.execute_reply":"2021-07-17T18:57:10.121642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## RUN TRAIN","metadata":{}},{"cell_type":"code","source":"if TRAIN:\n    model,history=run_train_validation_epochs(model=model,optimizer=optimizer, train_loader=train_loader,valid_loader=validation_loader,num_epochs=NUMBER_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.124001Z","iopub.execute_input":"2021-07-17T18:57:10.124672Z","iopub.status.idle":"2021-07-17T18:57:10.136100Z","shell.execute_reply.started":"2021-07-17T18:57:10.124634Z","shell.execute_reply":"2021-07-17T18:57:10.135016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EVAlUATE LOSS x EPOCH","metadata":{}},{"cell_type":"code","source":"if TRAIN:\n    total_epochs=NUMBER_EPOCHS\n    epochs=list(range(1,total_epochs+1))\n    print(history['Train Loss'])\n    train_loss=history['Train Loss']\n    valid_loss=history['Valid Loss']\n    print(history['Valid Loss'])\n    plt.plot(epochs,train_loss,label = \"Train\")\n    plt.plot(epochs,valid_loss,label = \"Validation\")\n    plt.xlabel('epoch')\n    plt.ylabel('loss')\n    plt.title('Loss over epochs')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.137524Z","iopub.execute_input":"2021-07-17T18:57:10.138205Z","iopub.status.idle":"2021-07-17T18:57:10.146740Z","shell.execute_reply.started":"2021-07-17T18:57:10.138163Z","shell.execute_reply":"2021-07-17T18:57:10.145632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LOAD A TRAINED CHECKPOINT","metadata":{}},{"cell_type":"code","source":"if LOAD:\n    model.load_state_dict(torch.load(LOAD_CHECK_PATH,map_location=CONFIG.device))","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:10.148504Z","iopub.execute_input":"2021-07-17T18:57:10.149373Z","iopub.status.idle":"2021-07-17T18:57:12.335224Z","shell.execute_reply.started":"2021-07-17T18:57:10.149331Z","shell.execute_reply":"2021-07-17T18:57:12.334218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TRESHOLD EVALUTION","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score,accuracy_score,classification_report\ndef get_df_for_f1_in_validation(model,labels,dataloader):\n    df=pd.DataFrame(columns=['image',*labels])\n    \n    model.eval()\n    N = 4\n    ground_truth=[]\n    for img, y in dataloader:\n        y_hat = np.zeros((img.size(0),6))\n        img=img.to(CONFIG.device)\n        for _ in range(N):\n            y_hat += model(img).detach().cpu().numpy()\n        y_hat /= N\n        for tensor in y:\n            ground_truth.append(tensor.cpu().numpy())\n        for image,lbls in enumerate(y_hat):\n            df=df.append({'image':f'sample_{image}',**{l:v for l,v in zip(labels,lbls)}},ignore_index=True)\n    return df,ground_truth\n\ndef get_score_from_treshold(df,ground_truth,labels,tresholds):\n    extract_labels=[]\n    for label in labels:\n        df[label+'_']=df[label].apply(lambda x: 0 if x<thresholds[label] else 1)\n        extract_labels.append(label+'_')\n    pred=df[extract_labels].to_numpy()\n    score=f1_score(pred,ground_truth,average='micro')\n    return score\n\ndef get_score_and_accuracy_from_treshold(df,ground_truth,labels,tresholds):\n    extract_labels=[]\n    for label in labels:\n        df[label+'_']=df[label].apply(lambda x: 0 if x<thresholds[label] else 1)\n        extract_labels.append(label+'_')\n    pred=df[extract_labels].to_numpy()\n    \n    score_f1=f1_score(pred,ground_truth,average='micro')\n    score_acc=accuracy_score(pred,ground_truth,)\n    print(classification_report(pred,ground_truth,target_names=labels))\n    return score_f1,score_acc","metadata":{"execution":{"iopub.status.busy":"2021-07-17T19:07:54.216333Z","iopub.execute_input":"2021-07-17T19:07:54.216740Z","iopub.status.idle":"2021-07-17T19:07:54.235364Z","shell.execute_reply.started":"2021-07-17T19:07:54.216702Z","shell.execute_reply":"2021-07-17T19:07:54.230613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"treshold_value=0.5\n\nif EVALUATE_TRESHOLD:\n    labels=map_dictionary.keys()\n    df_f1,ground_truth=get_df_for_f1_in_validation(model,labels,validation_loader)\n    treshold_values=[0.4,0.5,0.6,0.7,0.8]\n    scores_f1=[]\n    best_f1=0\n    for value in treshold_values:\n        thresholds= [value]*6\n        thresholds={k:v for k,v in zip(labels, thresholds)}\n        score=get_score_from_treshold(df_f1,ground_truth,labels,thresholds)\n        scores_f1.append(score)\n        if(score > best_f1):\n            best_f1=score\n            treshold_value=value\n            \n    plt.plot(treshold_values,scores_f1)\n    plt.xlabel('treshold')\n    plt.ylabel('f1 score')\n    plt.title('F1 in validation x treshold')\n    plt.legend()\n    plt.show()\n    \ntreshold_value","metadata":{"execution":{"iopub.status.busy":"2021-07-17T18:57:12.412371Z","iopub.execute_input":"2021-07-17T18:57:12.412714Z","iopub.status.idle":"2021-07-17T18:59:28.842661Z","shell.execute_reply.started":"2021-07-17T18:57:12.412680Z","shell.execute_reply":"2021-07-17T18:59:28.840975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SUBMIT:\n    labels=map_dictionary.keys()\n    df_f1,ground_truth=get_df_for_f1_in_validation(model,labels,validation_loader)\n    thresholds= [treshold_value]*6\n    thresholds={k:v for k,v in zip(labels, thresholds)}\n    score_f1,score_acc=get_score_and_accuracy_from_treshold(df_f1,ground_truth,labels,thresholds)\n    print(f\"validation f1_score={score_f1} accuracy={score_acc}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-17T19:07:58.885013Z","iopub.execute_input":"2021-07-17T19:07:58.885380Z","iopub.status.idle":"2021-07-17T19:10:13.805744Z","shell.execute_reply.started":"2021-07-17T19:07:58.885348Z","shell.execute_reply":"2021-07-17T19:10:13.804744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CREATE SUBMISSION","metadata":{}},{"cell_type":"code","source":"if SUBMIT:\n    labels=map_dictionary.keys()\n    thresholds= [treshold_value]*6\n    thresholds={k:v for k,v in zip(labels, thresholds)}\n    thresholds\n    paths=os.listdir(base_test_dir)\n    test_loader = DataLoader(PlantDatasetTest(base_test_dir,paths), shuffle=False,num_workers=2,batch_size=CONFIG.train_batch_size)\n\n    df=pd.DataFrame(columns=['image',*labels])\n    \n    model.eval()\n    N = 4\n    for img, y in test_loader:\n        img=img.to(CONFIG.device)\n        y_hat = np.zeros((img.size(0),6))\n        for _ in range(N):\n            y_hat += model(img).detach().cpu().numpy()\n        y_hat /= N\n        for image,lbls in zip(y,y_hat):\n            df=df.append({'image':image,**{l:v for l,v in zip(labels,lbls)}},ignore_index=True)\n    for label in labels:\n        df[label+'_']=df[label].apply(lambda x: '' if x<thresholds[label] else label+' ')\n    \n    df['labels']=df.apply(lambda x: ''.join(x[7:13]),axis=1)\n    df.labels=df.labels.apply(lambda x: 'healthy' if x=='' else x)\n    df.labels=df.labels.apply(lambda x: 'healthy' if 'healthy' in x else x)\n    df[['labels','image']].to_csv('submission.csv',index=False)\n\n\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-17T19:01:45.779471Z","iopub.status.idle":"2021-07-17T19:01:45.779853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}