{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About the Notebook\n\n- This is my baseline pytorch implementation for the competition\n- For now it's only single fold with smaller images/lighter model training for timing purposes.\n\n\nIf you find this notebook, please don't forget to upvote :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"gpu_info = !nvidia-smi\ngpu_info = '\\n'.join(gpu_info)\nif gpu_info.find('failed') >= 0:\n    print('Select the Runtime > \"Change runtime type\" menu to enable a GPU accelerator, ')\n    print('and then re-execute this cell.')\nelse:\n    print(gpu_info)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/nfnets/pytorch-image-models-master')\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pylab import rcParams\nimport math\n\nimport os\nimport time\nimport cv2\nimport PIL.Image\nimport random\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim.optimizer import Optimizer\nfrom torch.optim.lr_scheduler import CosineAnnealingLR \nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm\nimport gc\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom warnings import filterwarnings\nfilterwarnings(\"ignore\")\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nplt.style.use('ggplot')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    \n    \"\"\"Seeding everything for consistent experiments...\"\"\"\n    \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    \n    n_splits = 5 \n    \n    fold_id = 0 # Fold to train\n\n    image_size = 224 \n    seed = 42\n    init_lr = 1e-4\n    batch_size = 64\n    valid_batch_size = 64\n    n_epochs = 15\n    num_workers = 8\n\n    use_amp = True  \n    early_stop = 5\n\n    model_name = 'vit_base_patch16_224'\n    train_dir = '../input/plant-pathology-2021-fgvc8/train.csv'\n    data_dir = '../input/plant-path-2021-256'\n    \n    target_size=12\n    \n    \nmodel_dir = f'weights/'\n! mkdir $model_dir","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Basic EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(CFG.train_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Counting target values.\n\ntarg_cts=train.labels.value_counts()\nfig = plt.figure(figsize=(12,6))\nsns.barplot(y=targ_cts.sort_values(ascending=False).index, x=targ_cts.sort_values(ascending=False).values, palette='summer')\nplt.title('Target Distribution')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\n\nle.fit(train.labels)\ntrain['labels'] = le.transform(train.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le_mapping = dict(zip(le.classes_, le.transform(le.classes_)))\nprint(le_mapping)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setting Folds"},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds['labels'])):\n    folds.loc[val_index, 'fold'] = int(n)\ntrain['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# applying some augmentations for regularizing effect\n\ntransforms_train = A.Compose([\n   A.RandomResizedCrop(CFG.image_size, CFG.image_size, scale=(0.85, 1), p=1), \n   A.HorizontalFlip(p=0.5),   \n   A.Transpose(p=0.5),            \n   A.VerticalFlip(p=0.5),\n   A.ShiftScaleRotate(p=0.5),\n  A.Normalize(\n         mean=[0.5, 0.5, 0.5],\n         std=[0.5, 0.5, 0.5], max_pixel_value=255.0, p=1.0),\n         ToTensorV2(p=1.0)\n])\n\ntransforms_valid = A.Compose([\n    A.Resize(CFG.image_size, CFG.image_size),\n    A.Normalize(\n         mean=[0.5, 0.5, 0.5],\n         std=[0.5, 0.5, 0.5], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# applying some augmentations for regularizing effect\n\ntransforms_train = A.Compose([\n   A.RandomResizedCrop(CFG.image_size, CFG.image_size, scale=(0.85, 1), p=1), \n   A.HorizontalFlip(p=0.5),   \n   A.Transpose(p=0.5),            \n   A.VerticalFlip(p=0.5),\n   A.ShiftScaleRotate(p=0.5),\n  A.Normalize(\n         mean=[0.5, 0.5, 0.5],\n         std=[0.5, 0.5, 0.5], max_pixel_value=255.0, p=1.0),\n         ToTensorV2(p=1.0)\n])\n\ntransforms_valid = A.Compose([\n    A.Resize(CFG.image_size, CFG.image_size),\n    A.Normalize(\n         mean=[0.5, 0.5, 0.5],\n         std=[0.5, 0.5, 0.5], max_pixel_value=255.0, p=1.0),\n    ToTensorV2(p=1.0)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loader"},{"metadata":{"trusted":true},"cell_type":"code","source":"class TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.file_names = df['image'].values\n        self.labels = df['labels'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{CFG.data_dir}/{file_name}'\n        image = cv2.imread(file_path)        \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading the images with augmentations\n\ntrain_dataset = TrainDataset(train, transform=transforms_train)\n\nfig, axs = plt.subplots(1, 5, figsize=(40,12))\n\nfor i in range(5):\n    image, label = train_dataset[i]\n    axs[i].imshow(image.T)\n    axs[i].title.set_text(f'Target Labels: {label}')\n\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train/Valid Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_func(train_loader):\n    \n    \"\"\" Main training function: Takes loaded images to predict labels, computes losses between predicted and training labels, clip gradients, return updated losses. \"\"\"\n    \n    model.train()\n    bar = tqdm(train_loader)\n    if CFG.use_amp:\n        scaler = torch.cuda.amp.GradScaler()\n    losses = []\n    scores = []\n    for batch_idx, (images, targets) in enumerate(bar):\n\n        images, targets = images.to(device), targets.to(device)\n        \n        if CFG.use_amp:           \n            with torch.cuda.amp.autocast():\n                preds = model(images)\n                loss = trn_criterion(preds, targets)\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n                \n        else:\n            output = model(images)\n            loss = trn_criterion(output, targets)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n        losses.append(loss.item())\n        bar.set_description(f'Mean Loss: {np.mean(losses):.5f}')\n\n    loss_train = np.mean(losses)\n    \n    return loss_train\n\n\ndef valid_func(valid_loader):\n    \n    \"\"\" Main validation function: Takes loaded images to predict labels, computes losses between predicted and valid labels, clip gradients, return updated losses. \"\"\"\n    \n    \n    model.eval()\n    bar = tqdm(valid_loader)\n\n    PROB = []\n    TARGETS = []\n    losses = []\n    PREDS = []\n    \n    with torch.no_grad():\n        for batch_idx, (images, targets) in enumerate(bar):\n\n            images, targets = images.to(device), targets.to(device)\n            output = model(images)\n            PREDS += [output.softmax(1).to('cpu').numpy().argmax(1)]\n            TARGETS += [targets.detach().to('cpu').numpy()]\n            loss = val_criterion(output, targets)\n            losses.append(loss.item())\n            bar.set_description(f'Loss: {loss.item():.5f}')   \n    TARGETS=np.concatenate(TARGETS)\n    PREDS=np.concatenate(PREDS)\n    f1_val = f1_score(TARGETS, PREDS, average='macro')\n    loss_valid = np.mean(losses)\n    return loss_valid, f1_val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Custom Model Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ViTModel(nn.Module):    \n   \n\n    def __init__(self, model_name=CFG.model_name, pretrained=False, target_size=CFG.target_size):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)            \n        #self.model.head = nn.Linear(self.model.head.in_features, target_size)\n        \n        self.custom_layers = nn.Sequential(nn.Linear(1000, 1000),\n                                           nn.ReLU(),\n                                           nn.BatchNorm1d(1000),\n                                           nn.Dropout(0.5),\n                                           nn.Linear(1000, 500),\n                                           nn.ReLU(),   \n                                           nn.BatchNorm1d(500),\n                                           nn.Dropout(0.5),\n                                           nn.Linear(500, CFG.target_size))\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.custom_layers(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ViTModel(pretrained=True)\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Settings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# setting criterions, optimizers, folds to train etc.\n\nval_criterion = nn.CrossEntropyLoss()\ntrn_criterion = nn.CrossEntropyLoss()\n\n# for sam optimizer you can change the base optimizer to get better results\n\n\noptimizer = torch.optim.Adam(model.parameters(),lr=CFG.init_lr)\n    \n\n    \n# here you can experiment with other schedulers too, they have decent impact on this competition\n\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, CFG.n_epochs, eta_min=1e-7)\n\n\ntrain_df_this = train[train['fold'] != CFG.fold_id]\ndf_valid_this = train[train['fold'] == CFG.fold_id]\n\ndataset_train = TrainDataset(train_df_this, transform=transforms_train)\ndataset_valid = TrainDataset(df_valid_this, transform=transforms_valid)\n\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=CFG.batch_size, shuffle=True,  num_workers=CFG.num_workers, drop_last=True, pin_memory=True)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# single fold training\n\nlog = {}\nf1_max = 0.\nloss_min = 99999\nnot_improving = 0\n\n\nfor epoch in range(1, CFG.n_epochs+1):\n    \n    \n    loss_train = train_func(train_loader)\n    loss_valid, f1 = valid_func(valid_loader)\n\n    log['loss_train'] = log.get('loss_train', []) + [loss_train]\n    log['loss_valid'] = log.get('loss_valid', []) + [loss_valid]\n    log['lr'] = log.get('lr', []) + [optimizer.param_groups[0][\"lr\"]]\n    log['f1'] = log.get('f1', []) + [f1]\n\n    content = time.ctime() + ' ' + f'Fold: 0, Epoch: {epoch}/{CFG.n_epochs}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, loss_train: {loss_train:.5f}, loss_valid: {loss_valid:.5f}, f1: {f1:.6f}.'\n    print(content)\n    not_improving += 1\n    \n    scheduler.step()\n    \n    if f1 > f1_max:\n        print(f'f1_max ({f1_max:.6f} --> {f1:.6f}). Saving model ...')\n        torch.save(model.state_dict(), f'{model_dir}{CFG.model_name}_fold{CFG.fold_id}_best_f1.pth')\n        f1_max = f1\n        not_improving = 0\n\n    if loss_valid < loss_min:\n        loss_min = loss_valid\n        torch.save(model.state_dict(), f'{model_dir}{CFG.model_name}_fold{CFG.fold_id}_best_loss.pth')\n        \n    if not_improving == CFG.early_stop:\n        print('Early Stopping...')\n        break\n        \n\n\ntorch.save(model.state_dict(), f'{model_dir}{CFG.model_name}_fold{CFG.fold_id}_final.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Notes\n\n### I created this notebook for baseline purposes, you can easily modify, improve this code to get better results. I might update some parts of the code when I have more GPU time available on kaggle, happy coding :)\n\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}