{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%%bash \ncp /kaggle/input/landmark-lib/efficientnet_pytorch-0.7.0.xyz /kaggle/working/efficientnet_pytorch-0.7.0.tar.gz\npip install /kaggle/working/efficientnet_pytorch-0.7.0.tar.gz\n\n# install rest of stuff\nfiles=$(ls /kaggle/input/landmark-lib/*.whl)\nfor file in $files\ndo\n    if [[ $file != *\"efficientnet\"* && $file != *\"Keras\"* ]]; then\n        pip install $file\n    fi\ndone\n\npip install pytorch-lightning","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport gc\ngc.enable()\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nimport multiprocessing as mp\nfrom functools import partial\nimport pickle\n\nimport cv2\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import _LRScheduler, StepLR\nimport torch_optimizer as optim\n\nimport pytorch_lightning as pl\n\n\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.simplefilter('ignore')\n%matplotlib inline\n\ntorch.backends.cudnn.benchmark = True\n\n# torch.cuda.memory_allocated()\n# torch.cuda.get_device_properties(0).total_memory / 1e9\nprint(torch.__version__, device, mp.cpu_count())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE = \"efficientnet-b0\"\nWEIGHTS_PATH = \"/kaggle/input/landmark-lib/efficientnet-b0-355c32eb.pth\"\nSIZE = (128, 128)\nEPOCHS = 4\nGRAD_ACCUMULATE = 2\nBS = 256\nLR_RANGE = [1e-7, 2e-4]\nMAX_GRP_NUM = 200\nMIN_LANDMARK_COUNT = 100\nSAMPLES_PER_GRP = 5\nNUM_TOP_PREDICTS = 20","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data\n### Get Training/ Validation dataframes\n- We ignore classes that have less than `MIN_LANDMARK_COUNT` examples.\n- Any class that has more than `MAX_GRP_NUM` we only take `MAX_GRP_NUM` samples.\n- We weight every example to count for the data imbalance."},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"/kaggle/input/landmark-recognition-2020/\"\ndf = pd.read_csv(path + \"train.csv\")\ndf[\"path\"] = df[\"id\"].map(lambda x: \"/\".join([path+\"train\"] + list(x[:3])+[x + \".jpg\"]))\ndf.sort_values(\"landmark_id\", inplace=True)\ndf.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counts = df[\"landmark_id\"].value_counts()\ntopk = counts[counts >= MIN_LANDMARK_COUNT].index\ndf = df[df[\"landmark_id\"].isin(topk)]\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = []\nfor _, grp in tqdm(df.groupby(\"landmark_id\")):\n    if len(grp) < MAX_GRP_NUM:\n        dfs.append(grp)\n    else:\n        dfs.append(grp.sample(MAX_GRP_NUM))\n\ndf = pd.concat(dfs)\nprint(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id2y = {id_:i for i, id_ in enumerate(df[\"landmark_id\"].unique())}\ndf[\"target\"] = df[\"landmark_id\"].map(lambda x: id2y[x])\ncounts = df[\"target\"].value_counts()\n\nweights = counts.max() / counts \ndf[\"weights\"] = weights.loc[df[\"target\"]].values\nprint(f\"There are {len(id2y)} classes\")\n\nprint(df.shape)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.15, random_state=42, stratify=df[\"target\"])\ntrain_df.reset_index(drop=True, inplace=True)\ntrain_df = train_df.sample(frac=1)\nval_df.reset_index(drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = []\nfor root, dirs, files in tqdm(os.walk(path+\"test/\")):\n    if files:\n        files = [root+\"/\"+file for file in files]\n        test_files.extend(files)\n        \ntest_df = pd.DataFrame({\"path\": test_files})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PyTorch Datasets + DataLoaders\n- We use Albumentations to augment the images. See below for the augmentations.\n- Read in and resize the images from cv2 since it is faster that way."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tfms = A.Compose([\n    A.RandomBrightnessContrast(),\n    A.Blur(),\n    A.RandomContrast(),\n    A.HorizontalFlip(),\n], p=0.9)\n\ntest_tfms = A.Compose([\n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    ToTensorV2()\n])\n\ntrain_tfms = A.Compose([train_tfms, test_tfms])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Images(Dataset):\n    def __init__(self, df: pd.DataFrame, tfms:A.Compose, train: bool = True):\n        \"\"\"\n        Parameters:\n            df (pd.DataFrame): DataFrame with data description\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n        \"\"\"\n        if train:\n            df[\"weights\"] = df[\"weights\"].astype(np.float32)\n        self.tfms = tfms\n        self.df = df.reset_index(drop=True)\n        self.train = train\n        \n    def __getitem__(self, index):\n        im_path = self.df.loc[index, 'path']\n        x = cv2.cvtColor(cv2.resize(cv2.imread(im_path), SIZE), cv2.COLOR_BGR2RGB)\n        x = self.tfms(image=x)['image']\n            \n        if self.train:\n            weights = self.df.loc[index, 'weights']\n            y = self.df.loc[index, 'target']\n            return x, y, weights\n        else:\n            return x\n    \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = Images(train_df, train_tfms)\nval_images = Images(val_df, test_tfms)\ntest_images = Images(test_df, test_tfms, train=False)\n\ntrain_dl = DataLoader(train_images, BS, num_workers=mp.cpu_count(), pin_memory=True, shuffle=True, drop_last=True)\nval_dl = DataLoader(val_images, BS, num_workers=mp.cpu_count(), pin_memory=True)\ntest_dl = DataLoader(test_images, BS, num_workers=mp.cpu_count(), pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss\nThe arcface loss is used instead of the usual cross entropy loss. See the arcface paper [here]().\n\nThe main points are:\n- We already get the cos of the angle between the head and the embedding. See model below for details.\n- We take the arccos to get the angle and add a margin m.\n- However, if the sum is greater than 180 degrees, we don't add that margin parameter. This is done because if we don't, the loss encourages the angle to get bigger (towards 180), which is the opposite of what we want."},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcFaceLoss(nn.Module):\n    def __init__(self, s:float=64.0, m:float=0.5):\n        super().__init__()\n        self.s, self.m = torch.tensor(s), torch.tensor(m)\n        self.threshold = torch.tensor(np.pi - m)\n        self.cross_entropy = partial(F.cross_entropy, reduction='none')\n        \n    def forward(self, costheta, y):\n        costheta_y = costheta[torch.arange(len(y)), y]\n        angle = torch.acos(costheta_y)\n        # ensure that new angle is less than pi before adding margin m\n        angle[angle < self.threshold] = angle[angle < self.threshold] + self.m\n        costheta_y = torch.cos(angle)\n        costheta[torch.arange(len(y)), y] = costheta_y.type(costheta.dtype)\n        \n        return self.cross_entropy(self.s*costheta, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model\nThe model that we are using is very similar to simly putting a \"head\" ontop of a \"efficientnet\" architecture. However, the difference is we normalise the embedding created from the base model before multiplying with the head which is also normalised. Note: Normalising here means making the vectors unit length."},{"metadata":{"trusted":true},"cell_type":"code","source":"def metrics(y_pred, y, weights, k=5):\n    \"\"\"\n    Weighted accuracy and top-k accuracy\n    parameters:\n    - y_pred: predicted logits or probabilities \n    - y: Actual class\n    - weights: importance of each instance **must sum to one**\n    - k: number of categories to look for\n    \"\"\"\n    topk = y_pred.topk(k=k, dim=-1)[1] == y[:, None]\n    topk_acc = (weights * topk.any(dim=-1).float()).sum()\n    acc = (weights * topk[:,0].float()).sum()\n    return acc, topk_acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(pl.LightningModule):\n    def __init__(self, classes, loss_fn=ArcFaceLoss()):\n        super().__init__()\n\n        # EfficientNet\n        self.base = model = EfficientNet.from_pretrained(BASE, WEIGHTS_PATH)\n                \n        # Replace last layer\n        self.centers = nn.Parameter(torch.randn(self.base._fc.in_features, classes))\n        self.loss_fn = loss_fn\n    \n    def get_embedding(self, x):\n        pool = F.adaptive_avg_pool2d(self.base.extract_features(x), 1)\n        pool = pool.view(x.shape[0], -1)\n        \n        lens = torch.sqrt((pool**2).sum(dim=-1, keepdim=True))\n        return pool / lens\n    \n    def forward(self, x):\n        embeds = self.get_embedding(x)\n        \n        lens = torch.sqrt((self.centers**2).sum(dim=0, keepdim=True))\n        centers = self.centers / lens\n        \n        return embeds.matmul(centers)\n    \n    def get_loss_metrics(self, batch):\n        x, y, weights = batch\n        y_pred = self(x)\n        \n        loss_all = self.loss_fn(y_pred, y)\n        loss = (loss_all * weights).mean()\n        \n        acc, topk_acc = metrics(y_pred, y, weights)\n        \n        return loss, acc, topk_acc\n\n    def training_step(self, batch, batch_idx):\n        loss, acc, topk_acc = self.get_loss_metrics(batch)\n        self.log_dict({'train_loss': loss, 'train_acc': acc, 'train_topk': topk_acc}, prog_bar=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        loss, acc, topk_acc = self.get_loss_metrics(batch)\n        self.log_dict({'valid_loss': loss, 'valid_acc': acc, 'valid_topk': topk_acc}, prog_bar=True)\n    \n    def configure_optimizers(self):\n        optimizer = optim.RAdam(self.parameters(), lr=1e-2)\n        scheduler = StepLR(optimizer, step_size=100, gamma=0.9)\n        return [optimizer], [scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Model(len(id2y)) # , unfreeze=unfreeze\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initialise\nIt seems like a smart idea to initialise the final \"center\" weights so that they are the average of the embeddings for a given category."},{"metadata":{"trusted":true},"cell_type":"code","source":"samples_per_landmark = pd.concat([grp.sample(SAMPLES_PER_GRP) for _, grp in train_df.groupby(\"landmark_id\")])\nsample_images = Images(samples_per_landmark, test_tfms) \nsamples_dl = DataLoader(sample_images, BS, num_workers=mp.cpu_count(), pin_memory=True, shuffle=False, drop_last=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embeds = []\nwith torch.no_grad():\n    model.eval()\n    for i, (x, _, _) in tqdm(enumerate(samples_dl), total=len(samples_dl)):\n        x = x.to(device)\n        embeds.extend(model.get_embedding(x))\n        \nembeds = torch.stack(embeds)\n\n# get average directions\ncenters = embeds.view(-1, SAMPLES_PER_GRP, embeds.shape[-1]).mean(dim=1)\nlens = torch.sqrt((centers**2).sum(dim=1))\ncenters = centers / lens[:, None]\nmodel.centers.data = centers.T","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = pl.Trainer(tpu_cores=8, val_check_interval=0.5) #\ntrainer.fit(model, train_dl, val_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(losses)\nplt.title(\"Loss\")\nplt.show()\nplt.plot(accs)\nplt.title(\"Accuracy\")\nplt.show()\nplt.plot(topk_accs)\nplt.title(\"Top-k\")\nplt.show()\n\nplt.plot(val_losses)\nplt.title(\"Validation Loss\")\nplt.show()\nplt.plot(val_accs)\nplt.title(\"Validation Accuracy\")\nplt.show()\nplt.plot(val_topk_accs)\nplt.title(\"Validation Top-k\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), \"model.ckpt\")\nwith open(\"id2y.pickle\", \"wb\") as f:\n    pickle.dump(id2y, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ps = []\ncategories = []\nwith torch.no_grad():\n    model.eval()\n    for x in tqdm(test_dl):\n        x = x.to(device)\n        output = F.softmax(64*model(x), -1)\n        p, category = torch.topk(output, NUM_TOP_PREDICTS)\n        ps.extend(p)\n        categories.extend(category)\n\nps = torch.stack(ps).cpu().numpy()\ncategories = torch.stack(categories).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y2id = {y:id_ for id_,y in id2y.items()}\ncategories = np.array([[y2id[pred] for pred in preds] for preds in categories])\ndef concat(label: np.ndarray, conf: np.ndarray) -> str:\n    return ' '.join([f'{L} {c}' for L, c in zip(label, conf)])\n\nlandmarks = [concat(category, p) for category, p in zip(categories, ps)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"id\"] = test_df[\"path\"].map(lambda x: os.path.basename(x)[:-4])\ntest_df[\"landmarks\"] = landmarks\ntest_df.drop(\"path\", axis=1).to_csv(\"/kaggle/working/submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"landmark\"] = test_df[\"landmarks\"].map(lambda x: int(x.split()[0]))\ntest_df[\"landmark_p\"] = test_df[\"landmarks\"].map(lambda x: float(x.split()[1]))\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Histogram of confidences of the top category. Interesting to see that the peak is just above >0.01. It's ok considering there are ~2000 categories."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(test_df[\"landmark_p\"], 40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Examples of highly confident test images\n- Left column is a sample of 5 images from training set\n- Right column is a sample of UPTO 5 images from the test set."},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (name, grp) in enumerate(test_df[test_df[\"landmark_p\"]>0.02].groupby(\"landmark\")):\n    if i == 5:\n        break\n    train_examples = df[df[\"landmark_id\"]==name].sample(4)[\"path\"].values\n    test_examples = grp.head(4)[\"path\"].values\n    print(name)\n    print(\"=\"*100)\n    plt.figure(figsize=(12, 12))\n    for j in range(4):\n        if len(train_examples) > j:\n            im_path = train_examples[j]\n            plt.subplot(int(f\"42{2*j+1}\"))\n            plt.imshow(cv2.cvtColor(cv2.resize(cv2.imread(im_path), SIZE), cv2.COLOR_BGR2RGB))\n        if len(test_examples) > j:\n            im_path = test_examples[j]\n            plt.subplot(int(f\"42{2*j+2}\"))\n            plt.imshow(cv2.cvtColor(cv2.resize(cv2.imread(im_path), SIZE), cv2.COLOR_BGR2RGB))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}