{"cells":[{"metadata":{},"cell_type":"markdown","source":"## [PyTorch Training + Inference] EfficientNet Baseline\n"},{"metadata":{},"cell_type":"markdown","source":"### Settup Dependencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/landmark-lib","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/landmark-lib/efficientnet_pytorch-0.7.0.xyz /tmp/pip/cache/efficientnet_pytorch-0.7.0.tar.gz\n!cp ../input/landmark-lib/torch_optimizer-0.0.1a15-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/landmark-lib/pytorch_ranger-0.1.1-py3-none-any.whl /tmp/pip/cache/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /tmp/pip/cache/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-index --find-links /tmp/pip/cache/ efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-index --find-links /tmp/pip/cache/ pytorch_ranger","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-index --find-links /tmp/pip/cache/ torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# !pip install efficientnet_pytorch\n# !pip install torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport gc\ngc.enable()\nimport sys\nimport math\nimport json\nimport time\nimport random\nfrom glob import glob\nfrom datetime import datetime\n\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport multiprocessing\nfrom sklearn.preprocessing import LabelEncoder\n\nimport torch\nimport torchvision\nfrom torch import Tensor\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom tqdm import tqdm\n\nimport efficientnet_pytorch\n\nimport torch_optimizer as optim\nimport albumentations as A\n\nimport sklearn\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Configuration\n\n*Note: Lots of improvement can be done simply here. e.g.*\n\n* MIN SAMPLES PER CLASS - This variable is a threshold for total number of images in a class. If has class has less than this count then it will be discarded from training set.\n* BATCH SIZE            - The number of images in each training batch.\n* EPOCHS                - Total number of epochs."},{"metadata":{"trusted":true},"cell_type":"code","source":"IN_KERNEL = os.environ.get('KAGGLE_WORKING_DIR') is not None\nMIN_SAMPLES_PER_CLASS = 100\nBATCH_SIZE = 256\nNUM_WORKERS = multiprocessing.cpu_count()\nMAX_STEPS_PER_EPOCH = 15000\nNUM_EPOCHS = 10\nLOG_FREQ = 10\nNUM_TOP_PREDICTS = 10","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read Train and Test DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-recognition-2020/train.csv')\ntest = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\ntrain_dir = '../input/landmark-recognition-2020/train/'\ntest_dir = '../input/landmark-recognition-2020/test/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class ImageDataset(torch.utils.data.Dataset):\n    def __init__(self, dataframe: pd.DataFrame, image_dir:str, mode: str):\n        self.df = dataframe\n        self.mode = mode\n        self.image_dir = image_dir\n        \n        transforms_list = []\n        if self.mode == 'train':\n            # Increase image size from (64,64) to higher resolution,\n            # Make sure to change in RandomResizedCrop as well.\n            transforms_list = [\n                transforms.Resize((64,64)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomChoice([\n                    transforms.RandomResizedCrop(64),\n                    transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),\n                    transforms.RandomAffine(degrees=15, translate=(0.2, 0.2),\n                                            scale=(0.8, 1.2), shear=15,\n                                            resample=Image.BILINEAR)\n                ]),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ]\n        else:\n            transforms_list.extend([\n                # Keep this resize same as train\n                transforms.Resize((64,64)),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ])\n        self.transforms = transforms.Compose(transforms_list)\n\n    def __getitem__(self, index: int):\n        image_id = self.df.iloc[index].id\n        image_path = f\"{self.image_dir}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n        image = Image.open(image_path)\n        image = self.transforms(image)\n\n        if self.mode == 'test':\n            return {'image':image}\n        else:\n            return {'image':image, \n                    'target':self.df.iloc[index].landmark_id}\n\n    def __len__(self) -> int:\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(train, test, train_dir, test_dir):\n    counts = train.landmark_id.value_counts()\n    selected_classes = counts[counts >= MIN_SAMPLES_PER_CLASS].index\n    num_classes = selected_classes.shape[0]\n    print('classes with at least N samples:', num_classes)\n\n    train = train.loc[train.landmark_id.isin(selected_classes)]\n    print('train_df', train.shape)\n    print('test_df', test.shape)\n\n    # filter non-existing test images\n    exists = lambda img: os.path.exists(f'{test_dir}/{img[0]}/{img[1]}/{img[2]}/{img}.jpg')\n    test = test.loc[test.id.apply(exists)]\n    print('test_df after filtering', test.shape)\n\n    label_encoder = LabelEncoder()\n    label_encoder.fit(train.landmark_id.values)\n    print('found classes', len(label_encoder.classes_))\n    assert len(label_encoder.classes_) == num_classes\n\n    train.landmark_id = label_encoder.transform(train.landmark_id)\n\n    train_dataset = ImageDataset(train, train_dir, mode='train')\n    test_dataset = ImageDataset(test, test_dir, mode='test')\n\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,\n                              shuffle=False, num_workers=4, drop_last=True)\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,\n                             shuffle=False, num_workers=NUM_WORKERS)\n\n    return train_loader, test_loader, label_encoder, num_classes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"def radam(parameters, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n    if isinstance(betas, str):\n        betas = eval(betas)\n    return optim.RAdam(parameters,\n                      lr=lr,\n                      betas=betas,\n                      eps=eps,\n                      weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    ''' Computes and stores the average and current value '''\n    def __init__(self) -> None:\n        self.reset()\n\n    def reset(self) -> None:\n        self.val = 0.0\n        self.avg = 0.0\n        self.sum = 0.0\n        self.count = 0\n\n    def update(self, val: float, n: int = 1) -> None:\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GAP(predicts: torch.Tensor, confs: torch.Tensor, targets: torch.Tensor) -> float:\n    ''' Simplified GAP@1 metric: only one prediction per sample is supported '''\n    assert len(predicts.shape) == 1\n    assert len(confs.shape) == 1\n    assert len(targets.shape) == 1\n    assert predicts.shape == confs.shape and confs.shape == targets.shape\n\n    _, indices = torch.sort(confs, descending=True)\n\n    confs = confs.cpu().numpy()\n    predicts = predicts[indices].cpu().numpy()\n    targets = targets[indices].cpu().numpy()\n\n    res, true_pos = 0.0, 0\n\n    for i, (c, p, t) in enumerate(zip(confs, predicts, targets)):\n        rel = int(p == t)\n        true_pos += rel\n\n        res += true_pos / (i + 1) * rel\n\n    res /= targets.shape[0] # FIXME: incorrect, not all test images depict landmarks\n    return res","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model\n\n*Note: Used efficientnet-b0. Experimenting with different archs can yield different results*"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = efficientnet_pytorch.EfficientNet.from_pretrained(f'efficientnet-b{self.depth}',\"../input/landmark-lib/efficientnet-b0-355c32eb.pth\")\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.output_filter = self.base._fc.in_features\n        self.classifier = nn.Linear(640, num_classes)\n        print(\"filter size:\",self.output_filter)\n        self.layer1 = nn.Conv2d(self.output_filter,640,1)\n        # self.max = bb.\n    def forward(self, x):\n        x = self.base.extract_features(x)\n        x = self.avg_pool(x)\n        x = self.layer1(x).squeeze(-1).squeeze(-1)\n        x = self.classifier(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(train_loader, \n          model, \n          criterion, \n          optimizer,\n          epoch, \n          lr_scheduler):\n    print(f'epoch {epoch}')\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    avg_score = AverageMeter()\n\n    model.train()\n    num_steps = min(len(train_loader), MAX_STEPS_PER_EPOCH)\n\n    print(f'total batches: {num_steps}')\n\n    end = time.time()\n    lr = None\n    print(len(train_loader))\n    for i, data in enumerate(train_loader):\n        input_ = data['image']\n        target = data['target']\n        batch_size, _, _, _ = input_.shape\n        \n        output = model(input_.cuda())\n        loss = criterion(output, target.cuda())\n        confs, predicts = torch.max(output.detach(), dim=1)\n        avg_score.update(GAP(predicts, confs, target))\n        losses.update(loss.data.item(), input_.size(0))        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        lr_scheduler.step()\n        lr = optimizer.param_groups[0]['lr']\n        \n        batch_time.update(time.time() - end)\n        end = time.time()\n\n        if i % LOG_FREQ == 0:\n            print(f'{epoch} [{i}/{num_steps}]\\t'\n                    f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                    f'loss {losses.val:.4f} ({losses.avg:.4f})\\t'\n                    f'GAP {avg_score.val:.4f} ({avg_score.avg:.4f})'\n                    + str(lr))\n        \n\n    print(f' * average GAP on train {avg_score.avg:.4f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inference Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(data_loader, model):\n    model.eval()\n\n    activation = nn.Softmax(dim=1)\n    all_predicts, all_confs, all_targets = [], [], []\n\n    with torch.no_grad():\n        for i, data in enumerate(tqdm(data_loader, disable=IN_KERNEL)):\n            if data_loader.dataset.mode != 'test':\n                input_, target = data['image'], data['target']\n            else:\n                input_, target = data['image'], None\n            \n            output = model(input_.cuda())\n            output = activation(output)\n            \n            confs, predicts = torch.topk(output, NUM_TOP_PREDICTS)\n            all_confs.append(confs)\n            all_predicts.append(predicts)\n            if target is not None:\n                all_targets.append(target)\n            \n    predicts = torch.cat(all_predicts)\n    confs = torch.cat(all_confs)\n    targets = torch.cat(all_targets) if len(all_targets) else None\n\n    return predicts, confs, targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_submission(test_loader, model, label_encoder):\n    sample_sub = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n\n    predicts_gpu, confs_gpu, _ = inference(test_loader, model)\n    predicts, confs = predicts_gpu.cpu().numpy(), confs_gpu.cpu().numpy()\n\n    labels = [label_encoder.inverse_transform(pred) for pred in predicts]\n    print('labels')\n    print(np.array(labels))\n    print('confs')\n    print(np.array(confs))\n\n    sub = test_loader.dataset.df\n    def concat(label: np.ndarray, conf: np.ndarray) -> str:\n        return ' '.join([f'{L} {c}' for L, c in zip(label, conf)])\n    sub['landmarks'] = [concat(label, conf) for label, conf in zip(labels, confs)]\n\n    sample_sub = sample_sub.set_index('id')\n    sub = sub.set_index('id')\n    sample_sub.update(sub)\n\n    sample_sub.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Process"},{"metadata":{"trusted":true},"cell_type":"code","source":"global_start_time = time.time()\ntrain_loader, test_loader, label_encoder, num_classes = load_data(train, test, train_dir, test_dir)\n\nmodel = EfficientNetEncoderHead(depth=0, num_classes=num_classes)\nmodel.cuda()\n\ncriterion = nn.CrossEntropyLoss()\n\noptimizer = radam(model.parameters(), lr=1e-3, betas=(0.9,0.999), eps=1e-3, weight_decay=1e-4)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader)*NUM_EPOCHS, eta_min=1e-6)\n\nfor epoch in range(1, NUM_EPOCHS + 1):\n    print('-' * 50)\n    train_step(train_loader, model, criterion, optimizer, epoch, scheduler)\n\nprint('inference mode')\ngenerate_submission(test_loader, model, label_encoder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('inference mode')\ngenerate_submission(test_loader, model, label_encoder)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}