{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nThis notebook generates the trained model for https://www.kaggle.com/hdsk38/pytorch-starter-inference-efficientnet, which should give Public Score 0.003. Many codes are forked from https://www.kaggle.com/rhtsingh/pytorch-training-inference-efficientnet-baseline. Thank you [\ntorch](https://www.kaggle.com/rhtsingh)!\n\nSince the train data is big (1,580,470 images!!), I filtered the train data by selecting 10 images from each of the top 30,000 classes (out of 81,313 classes). Only 3 epochs were implemented in order to finish the training within a couple of hours. I hope this notebook will help you to join this competition :) !","metadata":{}},{"cell_type":"code","source":"!pip install ../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master/ > /dev/null # no output\n!pip install torch_optimizer --no-index --find-links=file:///kaggle/input/torch-optimizer/torch_optimizer","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:01.247171Z","iopub.execute_input":"2021-08-18T12:09:01.247597Z","iopub.status.idle":"2021-08-18T12:09:37.426605Z","shell.execute_reply.started":"2021-08-18T12:09:01.247514Z","shell.execute_reply":"2021-08-18T12:09:37.425589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\ngc.enable()\nimport sys\nimport math\nimport json\nimport time\nimport random\nfrom glob import glob\nfrom datetime import datetime\n\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport multiprocessing\nfrom sklearn.preprocessing import LabelEncoder\n\nimport torch\nimport torchvision\nfrom torch import Tensor\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom tqdm import tqdm\n\nimport efficientnet_pytorch\n\nimport torch_optimizer as optim\nimport albumentations as A\n\nimport sklearn\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:37.430260Z","iopub.execute_input":"2021-08-18T12:09:37.430562Z","iopub.status.idle":"2021-08-18T12:09:40.719563Z","shell.execute_reply.started":"2021-08-18T12:09:37.430532Z","shell.execute_reply":"2021-08-18T12:09:40.718676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IN_KERNEL = os.environ.get('KAGGLE_WORKING_DIR') is not None\nBATCH_SIZE = 64\nNUM_WORKERS = multiprocessing.cpu_count()\nMAX_STEPS_PER_EPOCH = 15000\nNUM_EPOCHS = 3\nLOG_FREQ = 10\nNUM_TOP_PREDICTS = 1","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:40.721417Z","iopub.execute_input":"2021-08-18T12:09:40.721771Z","iopub.status.idle":"2021-08-18T12:09:40.726694Z","shell.execute_reply.started":"2021-08-18T12:09:40.721735Z","shell.execute_reply":"2021-08-18T12:09:40.725894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/landmark-recognition-2021/train.csv')\ntrain_dir = '../input/landmark-recognition-2021/train/'","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:40.728379Z","iopub.execute_input":"2021-08-18T12:09:40.729052Z","iopub.status.idle":"2021-08-18T12:09:42.015879Z","shell.execute_reply.started":"2021-08-18T12:09:40.729013Z","shell.execute_reply":"2021-08-18T12:09:42.015011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(torch.utils.data.Dataset):\n    def __init__(self, dataframe: pd.DataFrame, image_dir:str, mode: str):\n        self.df = dataframe\n        self.mode = mode\n        self.image_dir = image_dir\n        \n        transforms_list = []\n        if self.mode == 'train':\n            transforms_list = [\n                transforms.Resize((64,64)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomChoice([\n                    transforms.RandomResizedCrop(64),\n                    transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),\n                    transforms.RandomAffine(degrees=15, translate=(0.2, 0.2),\n                                            scale=(0.8, 1.2), shear=15,\n                                            resample=Image.BILINEAR)\n                ]),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ]\n        else:\n            transforms_list.extend([\n                # Keep this resize same as train\n                transforms.Resize((64,64)),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ])\n        self.transforms = transforms.Compose(transforms_list)\n\n    def __getitem__(self, index: int):\n        image_id = self.df.iloc[index].id\n        image_path = f\"{self.image_dir}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n        image = Image.open(image_path)\n        image = self.transforms(image)\n\n        if self.mode == 'test':\n            return {'image':image}\n        else:\n            return {'image':image, \n                    'target':self.df.iloc[index].landmark_id}\n\n    def __len__(self) -> int:\n        return self.df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.017148Z","iopub.execute_input":"2021-08-18T12:09:42.017474Z","iopub.status.idle":"2021-08-18T12:09:42.028671Z","shell.execute_reply.started":"2021-08-18T12:09:42.017440Z","shell.execute_reply":"2021-08-18T12:09:42.027762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(train, train_dir):\n    print('Original train_df size:', train.shape)     \n    counts = train.landmark_id.value_counts()\n    \n    num_samples=10\n    num_classes = 30000\n    print(f'Select train data from top {num_classes} classes, {num_samples} samples of each class')\n    x=[]\n    y=[]\n    \n    for k,landmark_id in enumerate(tqdm(counts.index[:num_classes])):\n        x.extend(train[train['landmark_id']==landmark_id]['id'].sample(n=num_samples, random_state=1,replace=True).values)\n        y.extend([landmark_id]*num_samples)\n    train=pd.DataFrame({'id':x,'landmark_id':y})\n    print('Filtered train_df size:', train.shape)\n\n    #Encode target labels with value between 0 and n_classes-1\n    label_encoder = LabelEncoder() \n    label_encoder.fit(train.landmark_id.values)\n    \n    #Check if the number of the classes are ok\n    assert len(label_encoder.classes_) == num_classes \n\n    train.landmark_id = label_encoder.transform(train.landmark_id)\n\n    train_dataset = ImageDataset(train, train_dir, mode='train')\n\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,\n                              shuffle=True,\n                              num_workers=NUM_WORKERS,\n                              drop_last=True)\n\n    return train_loader, label_encoder, num_classes","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.030117Z","iopub.execute_input":"2021-08-18T12:09:42.030497Z","iopub.status.idle":"2021-08-18T12:09:42.055284Z","shell.execute_reply.started":"2021-08-18T12:09:42.030458Z","shell.execute_reply":"2021-08-18T12:09:42.054314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def radam(parameters, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n    if isinstance(betas, str):\n        betas = eval(betas)\n    return optim.RAdam(parameters,\n                      lr=lr,\n                      betas=betas,\n                      eps=eps,\n                      weight_decay=weight_decay)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.056940Z","iopub.execute_input":"2021-08-18T12:09:42.057403Z","iopub.status.idle":"2021-08-18T12:09:42.069736Z","shell.execute_reply.started":"2021-08-18T12:09:42.057360Z","shell.execute_reply":"2021-08-18T12:09:42.068708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter:\n    ''' Computes and stores the average and current value '''\n    def __init__(self) -> None:\n        self.reset()\n\n    def reset(self) -> None:\n        self.val = 0.0\n        self.avg = 0.0\n        self.sum = 0.0\n        self.count = 0\n\n    def update(self, val: float, n: int = 1) -> None:\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.072824Z","iopub.execute_input":"2021-08-18T12:09:42.073252Z","iopub.status.idle":"2021-08-18T12:09:42.080795Z","shell.execute_reply.started":"2021-08-18T12:09:42.073181Z","shell.execute_reply":"2021-08-18T12:09:42.079622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def GAP(predicts: torch.Tensor, confs: torch.Tensor, targets: torch.Tensor) -> float:\n    ''' Simplified GAP@1 metric: only one prediction per sample is supported '''\n    assert len(predicts.shape) == 1\n    assert len(confs.shape) == 1\n    assert len(targets.shape) == 1\n    assert predicts.shape == confs.shape and confs.shape == targets.shape\n\n    _, indices = torch.sort(confs, descending=True)\n\n    confs = confs.cpu().numpy()\n    predicts = predicts[indices].cpu().numpy()\n    targets = targets[indices].cpu().numpy()\n\n    res, true_pos = 0.0, 0\n\n    for i, (c, p, t) in enumerate(zip(confs, predicts, targets)):\n        rel = int(p == t)\n        true_pos += rel\n\n        res += true_pos / (i + 1) * rel\n\n    res /= targets.shape[0] # FIXME: incorrect, not all test images depict landmarks\n    return res","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.082520Z","iopub.execute_input":"2021-08-18T12:09:42.083024Z","iopub.status.idle":"2021-08-18T12:09:42.094728Z","shell.execute_reply.started":"2021-08-18T12:09:42.082987Z","shell.execute_reply":"2021-08-18T12:09:42.093704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = efficientnet_pytorch.EfficientNet.from_name(f'efficientnet-b{self.depth}')\n        pretrained_file = glob(f'../input/efficientnet-pytorch/efficientnet-b{self.depth}*')[0]\n        checkpoint = torch.load(pretrained_file)\n        self.base.load_state_dict(checkpoint)\n        \n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.output_filter = self.base._fc.in_features\n        self.classifier = nn.Linear(self.output_filter, num_classes)\n        \n    def forward(self, x):\n        x = self.base.extract_features(x)\n        x = self.avg_pool(x).squeeze(-1).squeeze(-1)\n        x = self.classifier(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.098502Z","iopub.execute_input":"2021-08-18T12:09:42.098837Z","iopub.status.idle":"2021-08-18T12:09:42.107990Z","shell.execute_reply.started":"2021-08-18T12:09:42.098773Z","shell.execute_reply":"2021-08-18T12:09:42.107007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(train_loader, \n          model, \n          criterion, \n          optimizer,\n          epoch, \n          lr_scheduler):\n    print(f'epoch {epoch}')\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    avg_score = AverageMeter()\n\n    model.train()\n    num_steps = min(len(train_loader), MAX_STEPS_PER_EPOCH)\n\n    print(f'total batches: {num_steps}')\n\n    end = time.time()\n    lr = None\n\n    for i, data in enumerate(train_loader):\n        input_ = data['image']\n        target = data['target']\n        batch_size, _, _, _ = input_.shape\n        \n        output = model(input_.cuda())\n        loss = criterion(output, target.cuda())\n        confs, predicts = torch.max(output.detach(), dim=1)\n        avg_score.update(GAP(predicts, confs, target))\n        losses.update(loss.data.item(), input_.size(0))\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        lr_scheduler.step()\n        lr = optimizer.param_groups[0]['lr']\n        \n        batch_time.update(time.time() - end)\n        end = time.time()\n\n        if i % LOG_FREQ == 0:\n            print(f'{epoch} [{i}/{num_steps}]\\t'\n                    f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n                    f'loss {losses.val:.4f} ({losses.avg:.4f})\\t'\n                    f'GAP {avg_score.val:.4f} ({avg_score.avg:.4f})'\n                    + str(lr))\n\n    print(f' * average GAP on train {avg_score.avg:.4f}')","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.109609Z","iopub.execute_input":"2021-08-18T12:09:42.110144Z","iopub.status.idle":"2021-08-18T12:09:42.122213Z","shell.execute_reply.started":"2021-08-18T12:09:42.110082Z","shell.execute_reply":"2021-08-18T12:09:42.121086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n    print(\"A\")\n    train_loader, label_encoder, num_classes = load_data(train, train_dir)\n    \n    print(\"B\")\n    model = EfficientNetEncoderHead(depth=0, num_classes=num_classes)\n    \n    print(\"C\")\n    model.cuda()\n    \n    print(\"D\")\n    criterion = nn.CrossEntropyLoss()\n    \n    print(\"E\")\n    optimizer = radam(model.parameters(), lr=1e-3, betas=(0.9,0.999), eps=1e-3, weight_decay=1e-4)\n    \n    print(\"F\")\n    scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader)*NUM_EPOCHS, eta_min=1e-6)\n    \n    print(\"G\")\n    for epoch in range(1, NUM_EPOCHS + 1):\n        print('-' * 50)\n        train_step(train_loader, model, criterion, optimizer, epoch, scheduler)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:09:42.123549Z","iopub.execute_input":"2021-08-18T12:09:42.123967Z","iopub.status.idle":"2021-08-18T12:15:17.483293Z","shell.execute_reply.started":"2021-08-18T12:09:42.123928Z","shell.execute_reply":"2021-08-18T12:15:17.480955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save classes of the label_encoder","metadata":{}},{"cell_type":"code","source":"np.save('classes.npy', label_encoder.classes_)","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:15:17.485010Z","iopub.status.idle":"2021-08-18T12:15:17.485860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save model weights","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), \"new_weight_efficientnet.pth\")","metadata":{"execution":{"iopub.status.busy":"2021-08-18T12:15:17.487436Z","iopub.status.idle":"2021-08-18T12:15:17.488166Z"},"trusted":true},"execution_count":null,"outputs":[]}]}