{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch Efficientnet + ArcFace [training]\n## Introduction\n* This notebook uses Efficient net and ArcFace on pytorch for training a model.\n* to train for 10 epoch I got validation accuracy about 0.25 and validation GAP score about 0.20\n* Inference notebook for submission is another one https://www.kaggle.com/zaccheroni/pytorch-efficientnet-arcface-submission Here!\n* To tell the truth I'm a beginer, so if there are any mistakes, please tell me!"},{"metadata":{},"cell_type":"markdown","source":"I used a notebook https://www.kaggle.com/rhtsingh/pytorch-training-inference-efficientnet-baseline by @rhtsingh - as a reference."},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/resources-for-google-landmark-recognition-2020/efficientnet_pytorch-0.6.3-py3-none-any.whl /tmp/pip/cache/\n!pip install --no-index --find-links /tmp/pip/cache/ efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if you need notification\n#!pip install slackweb\n# import slackweb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\ngc.enable()\nimport sys\nimport math\nimport json\nimport time\nimport random\nfrom glob import glob\nfrom datetime import datetime\n\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport multiprocessing\nfrom sklearn.preprocessing import LabelEncoder\n\nimport torch\nimport torchvision\nfrom torch import Tensor\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom tqdm import tqdm\n\nimport albumentations as A\n\nimport sklearn\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport csv\nimport pprint\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\nimport math\n\nfrom sklearn.model_selection import train_test_split\nimport torch.optim as optim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Seed everything to avoid non-determinism.\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    \nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IN_KERNEL = os.environ.get('KAGGLE_WORKING_DIR') is not None\nMIN_SAMPLES_PER_CLASS = 30\nBATCH_SIZE = 64\nNUM_WORKERS = multiprocessing.cpu_count()\nMAX_STEPS_PER_EPOCH = 15000\nNUM_EPOCHS = 8\nLOG_FREQ = 400\nNUM_TOP_PREDICTS = 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-recognition-2020/train.csv')\ntest = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\ntrain_dir = '../input/landmark-recognition-2020/train/'\ntest_dir = '../input/landmark-recognition-2020/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, val = train_test_split(train, test_size=0.02)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset(torch.utils.data.Dataset):\n    def __init__(self, dataframe: pd.DataFrame, image_dir:str, mode: str):\n        self.df = dataframe\n        self.mode = mode\n        self.image_dir = image_dir\n        \n        transforms_list = []\n        if self.mode == 'train':\n            # Increase image size from (64,64) to higher resolution,\n            # Make sure to change in RandomResizedCrop as well.\n            transforms_list = [\n                transforms.Resize((64,64)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomChoice([\n                    transforms.RandomResizedCrop(64),\n                    transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),\n                    transforms.RandomAffine(degrees=15, translate=(0.2, 0.2),\n                                            scale=(0.8, 1.2), shear=15,\n                                            resample=Image.BILINEAR)\n                ]),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ]\n        else:\n            transforms_list.extend([\n                # Keep this resize same as train\n                transforms.Resize((64,64)),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ])\n        self.transforms = transforms.Compose(transforms_list)\n\n    def __getitem__(self, index: int):\n        image_id = self.df.iloc[index].id\n        image_path = f\"{self.image_dir}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n        image = Image.open(image_path)\n        image = self.transforms(image)\n\n        if self.mode == 'test':\n            return {'image':image}\n        else:\n            return {'image':image, \n                    'target':self.df.iloc[index].landmark_id}\n\n    def __len__(self) -> int:\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(train, val, test, train_dir, test_dir):\n    counts = train.landmark_id.value_counts()\n    selected_classes = counts[counts >= MIN_SAMPLES_PER_CLASS].index\n    num_classes = selected_classes.shape[0]\n    print('classes with at least N samples:', num_classes)\n    all_val_count = val.shape[0]\n\n    train = train.loc[train.landmark_id.isin(selected_classes)]\n    val = val.loc[val.landmark_id.isin(selected_classes)]\n    print('train_df', train.shape)\n    print('val_df', val.shape)\n    print('test_df', test.shape)\n\n    # filter non-existing test images\n    exists = lambda img: os.path.exists(f'{test_dir}/{img[0]}/{img[1]}/{img[2]}/{img}.jpg')\n    test = test.loc[test.id.apply(exists)]\n    print('test_df after filtering', test.shape)\n\n    label_encoder = LabelEncoder()\n    label_encoder.fit(train.landmark_id.values)\n    print('found classes', len(label_encoder.classes_))\n    assert len(label_encoder.classes_) == num_classes\n\n    train.landmark_id = label_encoder.transform(train.landmark_id)\n    val.landmark_id = label_encoder.transform(val.landmark_id)\n\n    train_dataset = ImageDataset(train, train_dir, mode='train')\n    val_dataset = ImageDataset(val, train_dir, mode='train')\n    test_dataset = ImageDataset(test, test_dir, mode='test')\n\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,\n                              shuffle=True, num_workers=4, drop_last=True)\n    \n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE,\n                              shuffle=True, num_workers=4, drop_last=True)\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,\n                             shuffle=False, num_workers=NUM_WORKERS)\n\n    return train_loader, val_loader, test_loader, label_encoder, num_classes, all_val_count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def adam(parameters, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n    if isinstance(betas, str):\n        betas = eval(betas)\n    return optim.Adam(parameters,\n                      lr=lr,\n                      betas=betas,\n                      eps=eps,\n                      weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    ''' Computes and stores the average and current value '''\n    def __init__(self) -> None:\n        self.reset()\n\n    def reset(self) -> None:\n        self.val = 0.0\n        self.avg = 0.0\n        self.sum = 0.0\n        self.count = 0\n\n    def update(self, val: float, n: int = 1) -> None:\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GAP(predicts: torch.Tensor, confs: torch.Tensor, targets: torch.Tensor) -> float:\n    ''' Simplified GAP@1 metric: only one prediction per sample is supported '''\n    assert len(predicts.shape) == 1\n    assert len(confs.shape) == 1\n    assert len(targets.shape) == 1\n    assert predicts.shape == confs.shape and confs.shape == targets.shape\n\n    _, indices = torch.sort(confs, descending=True)\n\n    confs = confs.cpu().numpy()\n    predicts = predicts[indices].cpu().numpy()\n    targets = targets[indices].cpu().numpy()\n\n    res, true_pos = 0.0, 0\n\n    for i, (c, p, t) in enumerate(zip(confs, predicts, targets)):\n        rel = int(p == t)\n        true_pos += rel\n\n        res += true_pos / (i + 1) * rel\n\n    res /= targets.shape[0] # FIXME: incorrect, not all test images depict landmarks\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, train, label=False):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        if train:\n            one_hot = torch.zeros(cosine.size(), device='cuda')\n            one_hot.scatter_(1, label.cuda().view(-1, 1).long(), 1)\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)  # you can use torch.where if your torch.__version__ is 0.4\n        else:\n            output = cosine\n        output *= self.s\n\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = efficientnet_pytorch.EfficientNet.from_pretrained(f'efficientnet-b{self.depth}')\n        self.gem = GeM()\n        self.output_filter = self.base._fc.in_features\n        self.fc = nn.Linear(self.output_filter, 1000)\n        self.arcface = ArcMarginProduct(1000, num_classes)\n    def forward(self, x, label):\n        x = self.base.extract_features(x)\n        x = self.gem(x).squeeze()\n        x = self.fc(x)\n        if self.training:\n            x = self.arcface(x, self.training, label)\n        else:\n            x = self.arcface(x, self.training)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## validation\nThis 'val_step' caliculate a validation accuracy and GAP score"},{"metadata":{"trusted":true},"cell_type":"code","source":"def val_step(val_loader,\n        model,\n        criterion,\n        label_encoder,\n        all_val_count):\n    \n    val_losses = AverageMeter()\n    val_gap_score = AverageMeter()\n    val_acc = AverageMeter()\n    model.eval()\n    acc_count = 0\n    first = True\n    end = time.time()\n    for i, data in enumerate(val_loader):\n        input_ = data['image']\n        target = data['target']\n        batch_size, _, _, _ = input_.shape\n        \n        output = model(input_.cuda(), target.cuda())\n        confs, predicts = torch.max(output.detach(), dim=1)\n        \n        if first:\n            all_confs = confs\n            all_predicts = predicts\n            all_targets = target\n            first = False\n        else:\n            all_confs = torch.cat([all_confs, confs])\n            all_predicts = torch.cat([all_predicts, predicts])\n            all_targets = torch.cat([all_targets, target])\n\n    val_gap_score = GAP(all_predicts, all_confs, all_targets)\n    val_gap_score = val_gap_score * len(all_confs) / all_val_count\n    \n    for i, (c, p, t) in enumerate(zip(all_confs, all_predicts, all_targets)):\n        if p == t:\n            acc_count += 1\n                \n    acc = float(acc_count) / all_val_count\n    val_time = time.time() - end\n    return acc, val_gap_score, val_time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(train_loader, \n          model, \n          criterion, \n          optimizer,\n          epoch, \n          lr_scheduler):\n    print(f'epoch {epoch}')\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    avg_score = AverageMeter()\n\n    model.train()\n    num_steps = min(len(train_loader), MAX_STEPS_PER_EPOCH)\n\n    print(f'total batches: {num_steps}')\n\n    end = time.time()\n    lr = None\n\n    for i, data in enumerate(tqdm(train_loader)):\n        input_ = data['image']\n        target = data['target']\n        batch_size, _, _, _ = input_.shape\n        \n        output = model(input_.cuda(), target.cuda())\n        loss = criterion(output, target.cuda())\n        confs, predicts = torch.max(output.detach(), dim=1)\n        avg_score.update(GAP(predicts, confs, target))\n        losses.update(loss.data.item(), input_.size(0))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        lr_scheduler.step()\n        lr = optimizer.param_groups[0]['lr']\n        \n        batch_time.update(time.time() - end)\n        end = time.time()\n\n        if i % (num_steps//10) == 0:\n            acc, val_gap, val_time = val_step(val_loader, model, criterion, label_encoder, all_val_count)\n            print('validation time '+str(val_time))\n            print(f'{epoch} [{i}/{num_steps}]\\t'\n                    f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                    f'loss {losses.val:.4f} ({losses.avg:.4f})\\t'\n                    f'GAP {avg_score.val:.4f} ({avg_score.avg:.4f})\\t'\n                    f'val_acc {acc}\\t'\n                    f'val_GAP {val_gap:.4f}\\t'\n                 )\n            #slack = slackweb.Slack(url=\"~~~~~~\")\n            #slack.notify(text= f'{epoch} [{i}/{num_steps}]\\t'\n                    #f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                    #f'loss {losses.val:.4f} ({losses.avg:.4f})\\t'\n                    #f'GAP {avg_score.val:.4f} ({avg_score.avg:.4f})\\t'\n                    #f'val_acc {acc}\\t'\n                    #f'val_GAP {val_gap:.4f}\\t')\n\n    print(f' * average GAP on train {avg_score.avg:.4f}')\n    print(f' time {batch_time.sum:.4f}')\n    return avg_score.avg, losses.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(data_loader, model):\n    model.eval()\n\n    activation = nn.Softmax(dim=1)\n    all_predicts, all_confs, all_targets = [], [], []\n\n    with torch.no_grad():\n        for i, data in enumerate(tqdm(data_loader, disable=IN_KERNEL)):\n            if data_loader.dataset.mode != 'test':\n                input_, target = data['image'], data['target']\n            else:\n                input_, target = data['image'], None\n\n            output = model(input_.cuda())\n            output = activation(output)\n\n            confs, predicts = torch.topk(output, NUM_TOP_PREDICTS)\n            all_confs.append(confs)\n            all_predicts.append(predicts)\n\n            if target is not None:\n                all_targets.append(target)\n\n    predicts = torch.cat(all_predicts)\n    confs = torch.cat(all_confs)\n    targets = torch.cat(all_targets) if len(all_targets) else None\n\n    return predicts, confs, targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Before starting training, check if there is a trained model. If I have one, I will load it also with optimizer and scheduler"},{"metadata":{},"cell_type":"markdown","source":"I made \"selected_classes.csv\". This file shows which classes are selected, in another word, this model is classifying into these classes."},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"if __name__ == '__main__':\n    modelname = 'the_model'\n    input_dir = '../input/'\n    \n    global_start_time = time.time()\n    train_loader, val_loader, test_loader, label_encoder, num_classes, all_val_count = load_data(train, val, test, train_dir, test_dir)\n\n    all_classes = label_encoder.classes_\n    all_classes = list(all_classes)\n    selected_classes = train.landmark_id\n    with open('selected_classes.csv', 'w') as f:\n        writer = csv.writer(f)\n        writer.writerow(all_classes)\n    \n    model = EfficientNetEncoderHead(depth=7, num_classes=num_classes)\n    model.cuda()\n\n    criterion = nn.CrossEntropyLoss()\n\n    optimizer = adam(model.parameters(), lr=1e-3, betas=(0.9,0.999), eps=1e-3, weight_decay=1e-4)\n    scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader)*NUM_EPOCHS, eta_min=1e-6)\n\n    s = False\n    # if there is 'learning.txt', read it and start training from the epoch which is written in that file.\n    if os.path.exists(input_dir + 'learning.txt'):\n        with open(input_dir + 'learning.txt') as f:\n            s = f.read()\n            \n    # opttimizer saving dir\n    opt_shc_path = 'optimizer_and_scheduler'\n        \n    \n    if s:\n        model.load_state_dict(torch.load(input_dir + 'the_model'+s+'.pth'))\n        start_epoch = int(s) + 1\n        checkpoint = torch.load(input_dir + 'optimizer_and_scheduler')\n        optimizer.load_state_dict(checkpoint['optimizer'])\n        scheduler.load_state_dict(checkpoint['scheduler'])\n        print('optimizer and scheduler are loaded')\n        \n        pre_history = pd.read_csv(input_dir + 'the_model_history.csv')\n    else:\n        pre_history = pd.DataFrame(columns=['epoch', 'GAP', 'loss'])\n        start_epoch = 1\n\n        \n    for epoch in range(start_epoch, NUM_EPOCHS + 1):\n        print('-' * 50)\n        score, loss = train_step(train_loader, model, criterion, optimizer, epoch, scheduler)\n        pre_history = pre_history.append({'GAP':score,'epoch':epoch,'loss':loss}, ignore_index=True)\n        \n        model_path = 'the_model'+str(epoch)+'.pth'\n        state = {\n            'optimizer': optimizer.state_dict(),\n            'scheduler': scheduler.state_dict()\n        }\n        torch.save(model.state_dict(), model_path)\n        torch.save(state, opt_shc_path)\n        \n        with open('learning.txt', mode='w') as f:\n            f.write(str(epoch))\n            \n        acc, val_gap, _ = val_step(val_loader, model, criterion, label_encoder, all_val_count)\n        \n        pre_history.to_csv('the_model_history.csv')\n            \n        # if you want to know about learning on slack\n        #slack = slackweb.Slack(url=\"~~~\")\n        #slack.notify(text= f'{epoch:.4f} epoch finished\\t'\n        #            f'val_acc {acc}\\t'\n        #            f'val_GAP {val_gap:.4f}\\t')\n        \n    #slack.notify(text= 'all learning finished')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}