{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Baseline Model Starter Code**\n\n\nThis notebook provides some basic starter code to train on the Herbarium 2022 Kaggle Competition from the competition hosts. \nSpecifically, this code provides data loaders for PyTorch and does standard normalization and augmentations on the training data. \nThe code then performs training over 4 epochs starting with a pretrained Resnet101 architecture, using CrossEntropy as the loss function, and SGD/ReduceOnPlateau as the optimizer/learning rate scheduler. \nAdditionally, this network employs the trick of proxy batch size to boost the effective batch size of optimization while using minimal GPU memory. \n\nThis notebook also gives simple code to use a saved model to generate a submission for this competition which obtains ~27 percent accuracy on the public data set. ","metadata":{}},{"cell_type":"markdown","source":"Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport torch\nimport pickle\nimport numpy as np\nfrom tqdm import tqdm\nimport torch.nn as nn\nfrom torch import optim\nfrom datetime import datetime\nfrom torch.optim import lr_scheduler\nfrom sklearn.metrics import f1_score\nfrom tensorboardX import SummaryWriter\nfrom torch.optim.lr_scheduler import MultiStepLR, CosineAnnealingLR,ReduceLROnPlateau\nimport torchvision\nfrom PIL import Image\nfrom torch.utils import data\nimport torchvision.transforms as transforms","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-02T22:32:26.180335Z","iopub.execute_input":"2022-03-02T22:32:26.180617Z","iopub.status.idle":"2022-03-02T22:32:26.186281Z","shell.execute_reply.started":"2022-03-02T22:32:26.180586Z","shell.execute_reply":"2022-03-02T22:32:26.185562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define some data util functions and loaders","metadata":{}},{"cell_type":"code","source":"\ndef generate_img_pth_class(data_dir,split=\"train\"):\n    \"\"\" Read meta-data and change to file and class list\"\"\"\n    # 1. Load json meta data\n    with open(os.path.join(data_dir,\"{}_metadata.json\".format(split))) as f:\n        file_data = json.load(f)\n    # 2. Iterate through image and class list and save full pth and class\n    print(\"Generating {} file and class list\".format(split))\n    if split == \"train\":\n        full_file_list, class_list = [],[]\n        for i in tqdm(range(len(file_data[\"annotations\"]))):\n            # Ensure same picture\n            assert file_data[\"annotations\"][i][\"image_id\"] == file_data[\"images\"][i][\"image_id\"]\n            full_file_list.append(os.path.join(data_dir,\"train_images\",file_data[\"images\"][i][\"file_name\"]))\n            class_list.append(file_data[\"annotations\"][i][\"category_id\"])\n        # 3. Return as np array\n        return np.array(full_file_list), np.array(class_list)\n    else:\n        full_file_list = []\n        full_id_list = []\n        for i in tqdm(range(len(file_data))):\n            full_id_list.append(file_data[i]['image_id'])\n            full_file_list.append(os.path.join(data_dir,\"test_images\",file_data[i][\"file_name\"]))\n        # 3. Return as np array\n        return np.array(full_file_list), np.array(full_id_list)\n\n\n\n\ndef get_loaders(data_dir,batch_size=32,num_workers=2):\n    \"\"\"Returns train and test loader\"\"\"\n    # 1. Get train data, make data set, and make data loader\n    train_file_pths, train_cls = generate_img_pth_class(data_dir)\n    train_data_set = HerbariumDataLoader(train_file_pths, train_cls)\n    train_loader = torch.utils.data.DataLoader(train_data_set,batch_size=batch_size,shuffle=True,pin_memory=True,num_workers=num_workers)\n    # 2. Calculate number of classes -- use additional 3 classes so that mapping is 1 to 1\n    num_classes = np.unique(train_cls)[-1]+1\n\n    return train_loader, num_classes\n\ndef get_transforms(train):\n    mean= [0.485, 0.456, 0.406]\n    std = [0.229, 0.224, 0.225]\n    if train:\n        transform = transforms.Compose([\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(15),\n            transforms.ToTensor(),\n            transforms.Normalize(mean, std)\n        ])\n    else:\n        transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean, std)\n        ])\n    return transform\n\nclass HerbariumDataLoader(data.Dataset):\n\n    def __init__(self, file_pths, class_list, train=True):\n        self.is_train_set = train\n        self.file_paths = file_pths\n        self.classes = class_list\n        self.transforms = get_transforms(train)\n        self.total_img_count = self.file_paths.shape[0]\n        self.image_size = 380\n\n    def __len__(self):\n        return self.total_img_count\n\n    def __getitem__(self,idx):\n        # 1. Get class\n        class_label = np.int64(self.classes[idx]) if self.is_train_set else None\n\n        # 2. Load image, resize and transform\n        img = Image.open(self.file_paths[idx])\n        img = img.resize((self.image_size,self.image_size))\n        img = self.transforms(img)\n\n        # 3. Return image and class\n        return img, class_label\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:31:35.126358Z","iopub.execute_input":"2022-03-02T22:31:35.126756Z","iopub.status.idle":"2022-03-02T22:31:35.146632Z","shell.execute_reply.started":"2022-03-02T22:31:35.126722Z","shell.execute_reply":"2022-03-02T22:31:35.145827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Setup logging for tensorboard, and F1 score","metadata":{}},{"cell_type":"code","source":"def log_metrics(metric_dict,epoch,title):\n    # log all metrics\n    for key in metric_dict.keys():\n        label_text = \"{}/{}\".format(title,key)\n        logger.add_scalar(label_text,metric_dict[key],epoch)\n\ndef calculate_f1(preds,labs):\n    preds = preds.detach().cpu().numpy()\n    labs = labs.cpu().numpy()\n\n    f1_vals = f1_score(preds,labs,average=\"weighted\")\n    return f1_vals\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:31:37.299068Z","iopub.execute_input":"2022-03-02T22:31:37.299333Z","iopub.status.idle":"2022-03-02T22:31:37.305347Z","shell.execute_reply.started":"2022-03-02T22:31:37.299303Z","shell.execute_reply":"2022-03-02T22:31:37.304532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Making train method for single epoch","metadata":{}},{"cell_type":"code","source":"def train():\n    # 1. Set network to be in training mode\n    net.train()\n    # 2. Setup loggers, and progress bar\n    correct_class = 0\n    total_loss = 0\n    total_num = 0\n    pbar = tqdm(enumerate(train_data_loader), total=len(train_data_loader), desc='Train epoch {}'.format(epoch))\n    # 3. Iterate through\n    for batch_idx, (data, class_lab) in pbar:\n        # 3a. Set to correct device\n        data, class_lab = data.to(device),class_lab.to(device)\n        # 3b. Forward propagate\n        class_output  = net(data)\n        # 3d. Calculate loss (averaged over pseudo batch size), and Backpropagate on fullbatch\n        loss_val = loss_fn(class_output, class_lab) / proxy_batch_size\n        loss_val.backward()\n        if (batch_idx+1) % int(proxy_batch_size) == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n        # 3f. Update loggers\n        _, predicted = torch.max(class_output.data, 1)\n        correct_class += (predicted == class_lab).sum().item()\n        total_loss += loss_val.item()\n        total_num += data.shape[0]\n        # 3g. Update progress bar\n        pbar.set_postfix(avg_acc=float(correct_class)/total_num)\n    # 4. Calculate average loss and accuracy and return\n    class_acc = float(correct_class)/total_num\n    avg_loss = total_loss/(batch_idx+ 1) # Average per batch\n\n    return {\"train_avg_loss\": avg_loss, \"class_acc\":class_acc}\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:31:39.319591Z","iopub.execute_input":"2022-03-02T22:31:39.319889Z","iopub.status.idle":"2022-03-02T22:31:39.32877Z","shell.execute_reply.started":"2022-03-02T22:31:39.31986Z","shell.execute_reply":"2022-03-02T22:31:39.328031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Method to create pretrained feature network, with untrained classification head","metadata":{}},{"cell_type":"code","source":"def build_net():\n    pretrained_model = torchvision.models.resnet101(pretrained=True)\n    pretrained_dict = pretrained_model.state_dict()\n    net = torchvision.models.resnet101(pretrained=False,num_classes=num_classes)\n\n    # Fiter out unneccessary keys\n    copy_pretrained_dict = pretrained_dict.copy()\n    for k, v in pretrained_dict.items():\n        if \"fc\" in k:\n            del copy_pretrained_dict[k]\n    net.load_state_dict(copy_pretrained_dict,strict=False)\n    return net\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:31:44.95096Z","iopub.execute_input":"2022-03-02T22:31:44.951206Z","iopub.status.idle":"2022-03-02T22:31:44.957061Z","shell.execute_reply.started":"2022-03-02T22:31:44.95118Z","shell.execute_reply":"2022-03-02T22:31:44.956262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train Model","metadata":{}},{"cell_type":"code","source":"# 1. Seed all processes\nRANDOM_SEED = 1994\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed(RANDOM_SEED)\n\n# 2. Setup device\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Using device: {}\".format(device))\nif device == \"cuda\":\n    torch.backends.cudnn.benchmark = True\n\n# 3. Setup logging\ndt_string = datetime.now().strftime(\"%d%m%Y%H%M%S\")\ndir_name = \"baseline_run/run_id_{}\".format(dt_string)\nlogger = SummaryWriter(log_dir=dir_name)\n\n# 4. Load data\nbatch_size = 16\nproxy_batch_size = (256.0/batch_size)\ndata_dir = \"/kaggle/input/herbarium-2022-fgvc9\" # PATH TO DATA \n\ntrain_data_loader, num_classes = get_loaders(data_dir,batch_size=batch_size)\n\n# 5. Load model, using pretrained imagenet weights, and put on device\nnet = build_net()\nnet.to(device)\n\n# 6.  Setup optimization, learning rate scheduler, and loss function\nnum_epochs = 4\noptimizer = optim.SGD(net.parameters(), lr=.6,momentum=.9,weight_decay=0.0001,nesterov=True)\nlr_schedule = ReduceLROnPlateau(optimizer)\nloss_fn = nn.CrossEntropyLoss()\n\n\n# 7. Start train/test loop\nbest_acc = 0\nlogging_dict = {}\nfor epoch in range(num_epochs):\n    # Train one epoch, get loss and acc\n    train_dict = train()\n    # Increment scheduler\n    lr_schedule.step(train_dict[\"train_avg_loss\"])\n    # Log all to tensorboard and logging dict\n    log_metrics(train_dict,epoch,\"full_exp\")\n    logging_dict[\"{}\".format(epoch)] = train_dict\n    # Save model every epoch (could ensemble later)\n    model_name = \"epoch_{}_ca_{:.3f}.pth\".format(epoch,train_dict[\"class_acc\"])\n    print(model_name)\n    # Save model name if it is the best\n    if train_dict[\"class_acc\"] > best_acc:\n        best_model_pth = os.path.join(dir_name,model_name)\n        best_acc = train_dict[\"class_acc\"]\n        torch.save(net.state_dict(),best_model_pth)\n\n\n\n\n\n# Close logger, and save logging dictionary\nlogger.close()\noutput = open(os.path.join(dir_name,\"full_logs.pkl\"), 'wb')\npickle.dump(logging_dict, output)\noutput.close()","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:32:31.560604Z","iopub.execute_input":"2022-03-02T22:32:31.561425Z","iopub.status.idle":"2022-03-02T22:33:14.288008Z","shell.execute_reply.started":"2022-03-02T22:32:31.561366Z","shell.execute_reply":"2022-03-02T22:33:14.286214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now using saved model make submission CSV","metadata":{}},{"cell_type":"code","source":"def get_test_loaders(data_dir,batch_size=32,num_workers=2):\n    \"\"\"Returns train and test loader\"\"\"\n    # 1. Get train data, make data set, and make data loader\n    test_file_pths, test_ids = generate_img_pth_class(data_dir,split=\"test\")\n    train_data_set = HerbariumTestDataLoader(test_file_pths, test_ids)\n    test_loader = torch.utils.data.DataLoader(train_data_set,batch_size=batch_size,shuffle=True,pin_memory=True,num_workers=num_workers)\n\n    return test_loader\n\nclass HerbariumTestDataLoader(data.Dataset):\n\n    def __init__(self, file_pths, id_list):\n        #self.is_train_set = train\n        self.file_paths = file_pths\n        self.ids = id_list\n        self.transforms = get_transforms(False)\n        self.total_img_count = self.file_paths.shape[0]\n        self.image_size = 380\n\n    def __len__(self):\n        return self.total_img_count\n\n    def __getitem__(self,idx):\n        # 1. Get id\n        id_label = np.int64(self.ids[idx])\n\n        # 2. Load image, resize and transform\n        img = Image.open(self.file_paths[idx])\n        img = img.resize((self.image_size,self.image_size))\n        img = self.transforms(img)\n\n        # 3. Return image and class\n        return img, id_label\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:33:18.532651Z","iopub.execute_input":"2022-03-02T22:33:18.533342Z","iopub.status.idle":"2022-03-02T22:33:18.544826Z","shell.execute_reply.started":"2022-03-02T22:33:18.533301Z","shell.execute_reply":"2022-03-02T22:33:18.543726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now produce prediction CSV","metadata":{}},{"cell_type":"code","source":"import csv\n# 4. Load data\nbatch_size = 32\ndata_dir = \"/kaggle/input/herbarium-2022-fgvc9\" # PATH TO DATA \n\n\ntest_data_loader = get_test_loaders(data_dir,batch_size=batch_size)\nnum_classes = 15505 # Actually there are only 15501 classes -- but to keep it a simpler 1 to 1 matching consider 4 extra classes\n# 5. Load model, using pretrained imagenet weights, and put on device\nnet = build_net()\nnet.to(device)\nMODEL_PTH = best_model_pth # PATH TO SAVED MODEL\nnet.load_state_dict(torch.load(MODEL_PTH))\nnet.eval()\n\ndef write_results(preds,id_lab):\n    for i in range(preds.shape[0]):\n        csv_writer.writerow(['{:d}'.format(int(id_lab[i])),'{:d}'.format(int(preds[i]))])\n\noutput_file = open(\"sample_submission.csv\", 'w')\ncsv_writer = csv.writer(output_file, delimiter=',')\ncsv_writer.writerow([\"Id\",\"Predicted\"])\n# Iterate through\npbar = tqdm(enumerate(test_data_loader), total=len(test_data_loader), desc='Inference')\n# 3. Iterate through\nwith torch.no_grad():\n    for batch_idx, (data, id_lab) in pbar:\n        # put to device\n        data = data.to(device)\n        # 3b. Forward propagate\n        class_output  = net(data)\n        # 3c. Get prediction\n        _, predicted = torch.max(class_output.data, 1)\n        # 3d. write to file\n        write_results(predicted.cpu().numpy(),id_lab.numpy())\noutput_file.close()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:33:30.150526Z","iopub.execute_input":"2022-03-02T22:33:30.151113Z","iopub.status.idle":"2022-03-02T22:33:45.529514Z","shell.execute_reply.started":"2022-03-02T22:33:30.15107Z","shell.execute_reply":"2022-03-02T22:33:45.52853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Future work\nThere are many ways to improve this starter code -- we provide some ideas below. \n## Data\nThe data can be augmented in more robust ways or ways that leverage the natural structure of the problem, and the image size could be increased. \nAdditionally, you could leverage off the hierarchical structure of the data, which we provide some starter code for below. \n## Model \nThe model could be improved by trying out different and bigger CNN and transformer models. Additionally, you could try things like using ArcFace to encourage stronger embeddings for each class/ stronger class separation. Additionally, for this dataset, the use of different types of learning rate schedulers may significantly affect accuracy. \n## Post-processing \nThere are many ways to post-process the output of your models using ensemble methods and multiple augmented predictions per image and ways to combine all those predictions creatively given the data set's constraints.\n\n\n","metadata":{}},{"cell_type":"code","source":"import torch\nfrom PIL import Image\nimport json\nimport pandas as pd\nimport torchvision\nimport numpy as np\nimport torch\nfrom torch.utils.data import DataLoader, WeightedRandomSampler\n\nclass HerbariumDataset(torch.utils.data.Dataset):\n    def __init__(self, basepath, train=True, hier=False, transform=torchvision.transforms.Compose([torchvision.transforms.ToTensor()])):\n        self.basepath=basepath\n        self.train=train\n        self.hier=hier\n        self.transform=transform\n            \n        if self.train :\n            metadata_path = basepath + 'train_metadata.json'\n            self.basepath = self.basepath + 'train_images/'\n        else:\n            metadata_path = basepath + 'test_metadata.json'\n            self.basepath = self.basepath + 'test_images/'\n            \n            \n        with open(metadata_path) as f:\n            metadata = json.load(f)\n        \n        if self.train:\n            self.df_images = pd.DataFrame(metadata['images'])\n        else:\n            self.df_images = pd.DataFrame(metadata)\n            \n        if self.train:\n            self.df_annotations = pd.DataFrame(metadata['annotations'])\n            self.df_images.index = self.df_images.image_id\n            self.df_categories = pd.DataFrame(metadata['categories'])\n\n        if self.hier:\n            genuses = list(set(self.df_categories.genus.values))\n            families = list(set(self.df_categories.family.values))\n            genuses.sort()\n            families.sort()\n            genuses2id = {item:i for item,i in zip(genuses,range(len(genuses)))}\n            families2id = {item:i for item,i in zip(families,range(len(families)))}\n            self.df_categories['genus_id'] = self.df_categories['genus'].map(genuses2id)\n            self.df_categories['family_id'] = self.df_categories['family'].map(families2id)\n            self.df_categories.index = self.df_categories['category_id']\n\n        \n    def __len__(self):\n        return len(self.df_images)\n    \n    def __getitem__(self, index):\n        label=0\n        \n        if self.train:       \n            \n            img_path = self.basepath+self.df_images.loc[self.df_annotations.loc[index].image_id].file_name\n            label = self.df_annotations.loc[index].category_id\n        \n            if self.hier:\n                family = self.df_categories.loc[label].family_id\n                genus = self.df_categories.loc[label].genus_id\n            \n\n        else:\n            img_path = self.basepath+self.df_images.loc[index].file_name\n       \n        img=self.image_reader(img_path)\n        if self.transform is not None:\n            img=self.transform(img)\n\n        if self.hier:\n            return img, np.array([family, genus, label]) \n        else:\n            return img,label\n\n    def image_reader(self,img_path):\n        img = Image.open(img_path).convert('RGB')\n        return img\n\n\n    def select_indices(self,indices):\n        self.df_annotations=self.df_annotations.loc[indices]\n        self.df_images=self.df_images.loc[self.df_annotations.image_id]\n        self.df_annotations.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-02T22:29:56.435157Z","iopub.execute_input":"2022-03-02T22:29:56.435522Z","iopub.status.idle":"2022-03-02T22:29:56.45582Z","shell.execute_reply.started":"2022-03-02T22:29:56.435489Z","shell.execute_reply":"2022-03-02T22:29:56.454545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}