Herbarium 2021 Pytorch Starter
"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"Install Required Packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Import Packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport copy\nimport time\nimport json\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda import amp\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, f1_score\nfrom sklearn.utils import class_weight\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n model_name = 'resnext50_32x4d'\n img_size = 224\n scheduler = 'CosineAnnealingLR'\n T_max = 5\n T_0 = 5\n lr = 1e-4\n min_lr = 1e-6\n batch_size = 128\n weight_decay = 1e-6\n seed = 42\n num_classes = 64500\n num_epochs = 1\n n_fold = 5\n device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_DIR = '../input/herbarium-2021-fgvc8/train/'\nTEST_DIR = '../input/herbarium-2021-fgvc8/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nwith open(TRAIN_DIR + 'metadata.json', \"r\", encoding=\"ISO-8859-1\") as file:\n train = json.load(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img = pd.DataFrame(train['images'])\ntrain_ann = pd.DataFrame(train['annotations']).drop(columns='image_id')\ndf = train_img.merge(train_ann, on='id')\n\nprint(len(df))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.category_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set Seed for Reproducibility"},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed = 42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create Folds"},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.category_id)):\n df.loc[val_ , \"kfold\"] = int(fold)\n \ndf['kfold'] = df['kfold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Augmentations & Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {\n \"train\": A.Compose([\n A.RandomResizedCrop(CFG.img_size, CFG.img_size),\n A.HorizontalFlip(p=0.5),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.),\n \n \"valid\": A.Compose([\n A.CenterCrop(CFG.img_size, CFG.img_size, p=1.),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.)\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dataset Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Herbarium2021(Dataset):\n def __init__(self, root_dir, df, transforms=None):\n self.root_dir = root_dir\n self.df = df\n self.transforms = transforms\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, idx):\n filename = self.df.iloc[idx, 0]\n image_path = os.path.join(TRAIN_DIR, filename)\n img = cv2.imread(image_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n label = self.df.iloc[idx, 5]\n \n if self.transforms is not None:\n img = self.transforms(image=img)[\"image\"]\n \n return img, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Resnext50_32x4d(nn.Module):\n def __init__(self, model_name='resnext50_32x4d', pretrained=True):\n super().__init__()\n self.model = timm.create_model(model_name, pretrained=pretrained)\n n_features = self.model.fc.in_features\n self.model.fc = nn.Linear(n_features, CFG.num_classes)\n\n def forward(self, x):\n x = self.model(x)\n return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold):\n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_f1 = 0.0\n history = defaultdict(list)\n scaler = amp.GradScaler()\n\n for epoch in range(1,num_epochs+1):\n print('Epoch {}/{}'.format(epoch, num_epochs))\n print('-' * 10)\n\n # Each epoch has a training and validation phase\n for phase in ['train','valid']:\n if(phase == 'train'):\n model.train() # Set model to training mode\n else:\n model.eval() # Set model to evaluation mode\n PREDS = []\n TARGETS = []\n \n running_loss = 0.0\n \n # Iterate over data\n for inputs,labels in dataloaders[phase]:\n inputs = inputs.to(CFG.device)\n labels = labels.to(CFG.device)\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n # forward\n # track history if only in train\n with torch.set_grad_enabled(phase == 'train'):\n with amp.autocast():\n outputs = model(inputs)\n _, preds = torch.max(outputs, 1)\n loss = criterion(outputs, labels)\n\n # backward + optimize only if in training phase\n if phase == 'train':\n scaler.scale(loss).backward()\n scaler.step(optimizer)\n scaler.update()\n \n if phase == 'valid':\n PREDS += [preds]\n TARGETS += [labels.detach().cpu()]\n\n running_loss += loss.item()*inputs.size(0)\n\n if phase == 'valid':\n PREDS = torch.cat(PREDS).cpu().numpy()\n TARGETS = torch.cat(TARGETS).cpu().numpy()\n epoch_f1 = f1_score(TARGETS, PREDS, average='weighted')\n history[phase + ' f1'].append(epoch_f1)\n \n epoch_loss = running_loss/dataset_sizes[phase]\n\n history[phase + ' loss'].append(epoch_loss)\n\n if phase == 'train' and scheduler != None:\n scheduler.step()\n\n print('{} Loss: {:.4f}'.format(\n phase, epoch_loss))\n \n if phase == 'valid':\n print('{} F1: {:.4f}'.format(\n phase, epoch_f1))\n \n # deep copy the model\n if phase == 'valid' and epoch_f1 >= best_f1:\n best_f1 = epoch_f1\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = f\"Fold_{fold}_{best_f1}_epoch{epoch}.bin\"\n torch.save(model.state_dict(), PATH)\n\n print()\n\n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best F1:\",best_f1)\n\n # load best model weights\n model.load_state_dict(best_model_wts)\n return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_fold(model, criterion, optimizer, scheduler, device, fold, num_epochs=10):\n valid_df = df[df.kfold == fold]\n train_df = df[df.kfold != fold]\n \n train_data = Herbarium2021(TRAIN_DIR, train_df, transforms=data_transforms[\"train\"])\n valid_data = Herbarium2021(TRAIN_DIR, valid_df, transforms=data_transforms[\"valid\"])\n \n dataset_sizes = {\n 'train' : len(train_data),\n 'valid' : len(valid_data)\n }\n \n train_loader = DataLoader(dataset=train_data, batch_size=CFG.batch_size, num_workers=4, \n pin_memory=True, shuffle=True, drop_last=True)\n valid_loader = DataLoader(dataset=valid_data, batch_size=CFG.batch_size, num_workers=4, \n pin_memory=True, shuffle=False, drop_last=False)\n \n dataloaders = {\n 'train' : train_loader,\n 'valid' : valid_loader\n }\n\n model, history = train_model(model, criterion, optimizer, scheduler, num_epochs, dataloaders, dataset_sizes, device, fold)\n \n return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CFG.scheduler == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr)\n elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr)\n elif CFG.scheduler == None:\n return None\n \n return scheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Resnext50_32x4d()\nmodel.to(CFG.device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\ncriterion = nn.CrossEntropyLoss()\nscheduler = fetch_scheduler(optimizer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Run Fold 0"},{"metadata":{"trusted":true},"cell_type":"code","source":"model, history = run_fold(model, criterion, optimizer, scheduler, device=CFG.device, fold=0, num_epochs=CFG.num_epochs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}