{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Reivew some method\nIn my experience the Image Retest Task has two main approaches: pair based loss and sofmax loss\n# 1. Based on pair-based loss::\n* **Contrastive**:\nThis method only considers 1 negative and 1 positive at the same time.\n* **Triplet loss**\nTriplet loss consider pair: anchor, positive, negative\nModel tries to push (anchor,negative) pair and pull (anchor, positive) pair -> The effectiveness of the method is based on batch size and it takes a long time to train && required strong GPU. Why ? In backpropagation it only considers 1 anchor (same class as positive), 1 postive and 1 negative -> so The method is difficult to converge.\nAnd some methods are suggested to reduce this disadvantage such as: [Lifted structured](https://arxiv.org/pdf/1511.06452.pdf), [N pair loss](https://proceedings.neurips.cc/paper/2016/file/6b180037abbebea991d8b1232f8a8ca9-Paper.pdf), ...\n\n# 2. Based on softmax loss:\nArcface, CosFace, Sphere, [Adaptive margin](https://openaccess.thecvf.com/content_CVPR_2019/html/Liu_AdaptiveFace_Adaptive_Margin_and_Sampling_for_Face_Recognition_CVPR_2019_paper.html)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"# SOTA 2021\n[Proxy anchor](https://arxiv.org/abs/2003.13911) is the strong method in image retrival task.\nThat outperform compare to famous method Constrative loss, triplet loss, NCA... and reduce vastly training time.\nThe author choose the anchor for each class ( not assing anchor to image), then the loss pull the same class and push another sample. Proxy anchor does not select pairs and or tuples but only considers anchor and its sample to pulls and pushes the other sample ( difference class of its), then PA does not need large GPU memory to achieve good results.","metadata":{}},{"cell_type":"code","source":"from torchvision import datasets\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nimport numpy as np\nfrom torch.utils.data import DataLoader,Dataset\nimport pandas as pd\nimport torchvision.models as models\nfrom PIL import Image\nimport tensorflow as tf\nfrom torch.utils.tensorboard import SummaryWriter\nfrom tqdm import tqdm\nimport torch.nn.init as init\nimport sklearn.preprocessing\nimport albumentations as A\nfrom torch.utils.data import DataLoader, Dataset\nfrom albumentations.pytorch import ToTensorV2\nimport cv2\nimport os\nimport random","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.165171Z","iopub.execute_input":"2022-02-05T16:12:14.165484Z","iopub.status.idle":"2022-02-05T16:12:14.172814Z","shell.execute_reply.started":"2022-02-05T16:12:14.165442Z","shell.execute_reply":"2022-02-05T16:12:14.172124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False # set True to be faster\n    print(f'Setting all seeds to be {seed} to reproduce...')\nseed_everything(1024)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.174519Z","iopub.execute_input":"2022-02-05T16:12:14.175498Z","iopub.status.idle":"2022-02-05T16:12:14.187505Z","shell.execute_reply.started":"2022-02-05T16:12:14.175457Z","shell.execute_reply":"2022-02-05T16:12:14.186492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config():\n    \n    path_model_pretrained = '/u01/sen/whale/pretrain_model/densenet121_ra-50efcf5c.pth'\n    path_info = '../input/happy-whale-and-dolphin/train.csv'\n    # path save file\n    model_name = 'densenet121' # densenet121, resnet50\n    loss_type = 'proxy_anchor'\n    optimizer = 'adam'\n    path_save_log = './proxy_anchor_412022/'\n    path_save_model = path_save_log + model_name\n    resume_model = ''\n    root_data_2022 = '../input/happy-whale-and-dolphin/train_images/'\n    phase_idx = 0 ## only on crop\n    \n# train\n    start_epoch = 0\n    train_number_epochs = 60\n    continue_train = False\n    batch_size = 64\n    num_embeddings = 512\n    input_size = 224\n    worker = 2\n    num_class = 29392 #11894 # 4019 #23786 #4019 #19767\n    scale_margin = 1\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    \n# optimize\n    lr = 1e-4\n    lr_decay_step = 30\n    lr_decay_gamma =  0.25\n    weight_decay = 1e-4\n    os.makedirs(path_save_log, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.189422Z","iopub.execute_input":"2022-02-05T16:12:14.190309Z","iopub.status.idle":"2022-02-05T16:12:14.240232Z","shell.execute_reply.started":"2022-02-05T16:12:14.190259Z","shell.execute_reply":"2022-02-05T16:12:14.239345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class min_edge_crop(A.ImageOnlyTransform):\n    \n    def __init__(self, p: float = 0.5, always_apply=True):\n        super().__init__(always_apply, p)\n        self.position = 'center'\n\n    def apply(self, img, **params):\n        \"\"\"\n        crop image base on min size\n        :param img: image to be cropped\n        :param position: where to crop the image\n        :return: cropped image\n        \"\"\"\n        assert self.position in ['center', 'left', 'right'], \"position must either be: left, center or right\"\n\n        h, w = img.shape[:2]\n\n        if h == w:\n            return img\n\n        min_edge = min(h, w)\n        if h > min_edge:\n            if self.position == \"left\":\n                img = img[:min_edge]\n            elif self.position == \"center\":\n                d = (h - min_edge) // 2\n                img = img[d:-d] if d != 0 else img\n\n                if h % 2 != 0:\n                    img = img[1:]\n            else:\n                img = img[-min_edge:]\n\n        if w > min_edge:\n            if self.position == \"left\":\n                img = img[:, :min_edge]\n            elif self.position == \"center\":\n                d = (w - min_edge) // 2\n                img = img[:, d:-d] if d != 0 else img\n\n                if w % 2 != 0:\n                    img = img[:, 1:]\n            else:\n                img = img[:, -min_edge:]\n        \n#         assert img.shape[0] == img.shape[1], f\"height and width must be the same, currently {img.shape[:2]}\"\n        return img\n\ndef get_augmentation(phase,input_size):\n    if phase == \"train\":\n        return  A.Compose([\n                    min_edge_crop(),\n                    A.Resize(height=input_size, width=input_size),\n                     A.ToGray(p=0.01),\n                     A.OneOf([\n                       A.GaussNoise(var_limit=[10, 50]),\n                       A.GaussianBlur(),\n                       A.MotionBlur(),\n                       A.MedianBlur(),\n                      ], p=0.2),\n                    A.OneOf([\n                       A.OpticalDistortion(distort_limit=1.0),\n                       A.GridDistortion(num_steps=5, distort_limit=1.),\n                       A.ElasticTransform(alpha=3),\n                   ], p=0.2),\n                     A.OneOf([\n                         A.CLAHE(),\n                         A.RandomBrightnessContrast(),\n                     ], p=0.25),\n                     A.HueSaturationValue(p=0.25),\n                    A.ShiftScaleRotate(p=0.5, shift_limit=0.0625, scale_limit=0.2, rotate_limit=20),\n#                     A.Cutout(max_h_size=int(input_size * 0.1), max_w_size=int(input_size * 0.1), num_holes=5, p=0.5),\n                    A.Normalize(),\n                    ToTensorV2()\n                ])\n    elif phase in ['test','valid']:\n        return A.Compose([\n            min_edge_crop(),\n            A.Resize(height=input_size, width=input_size),\n            A.Normalize(),\n            ToTensorV2()\n        ])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.24469Z","iopub.execute_input":"2022-02-05T16:12:14.245305Z","iopub.status.idle":"2022-02-05T16:12:14.265271Z","shell.execute_reply.started":"2022-02-05T16:12:14.24525Z","shell.execute_reply":"2022-02-05T16:12:14.264223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class whale_huback():\n    \n    def __init__(self, df, transform = None):\n        \n        self.df = df.reset_index()\n        self.transform = transform\n                \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self,index):\n        \n        img_path, class_id = self.df.loc[index, 'image'], self.df.loc[index,'individual_id']\n        sample = cv2.imread(img_path)\n        if sample is None:\n            print(img_path)\n            \n        sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)\n        if self.transform is not None:\n            sample = self.transform(image=sample)[\"image\"]\n\n        return sample, torch.tensor(class_id)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.269247Z","iopub.execute_input":"2022-02-05T16:12:14.269531Z","iopub.status.idle":"2022-02-05T16:12:14.280176Z","shell.execute_reply.started":"2022-02-05T16:12:14.2695Z","shell.execute_reply":"2022-02-05T16:12:14.279242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info = pd.read_csv(config.path_info)\ninfo['image'] = config.root_data_2022+info['image']\nprint(len(info['image']))\ninfo = info.sample(frac=1)\ninfo['source'] = '2022'\nmapping = {}\nindex=-1\nfor class_id in info['individual_id'].unique():\n    if class_id not in mapping.keys():\n        index+=1\n        mapping[class_id] = index\ninfo['individual_id'] = info['individual_id'].apply(lambda class_id: mapping[class_id]) \ntotal_class = info['individual_id'].unique().shape[0]\ninfo['phase'] = info['individual_id'].apply(lambda class_id: 'train' if class_id <= total_class//2 else 'valid')\ninfo[info['phase']=='train']['individual_id'].unique().shape[0]\ntrain = info[info['phase']=='train']\nvalid = info[info['phase']!='valid']\n\nprint('train.shape',train.shape)\nprint('valid.shape',train.shape)\ntrain.to_csv(\"source_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.417868Z","iopub.execute_input":"2022-02-05T16:12:14.418626Z","iopub.status.idle":"2022-02-05T16:12:14.858764Z","shell.execute_reply.started":"2022-02-05T16:12:14.418587Z","shell.execute_reply":"2022-02-05T16:12:14.85777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndataset = {\n        phase: whale_huback(eval(phase),transform = get_augmentation(phase=phase,input_size=config.input_size)) \\\n        for phase in ['train','valid']\n}\ndataloader = { phase: DataLoader(dataset=dataset[phase], num_workers=config.worker, batch_size=config.batch_size, \\\n                                          shuffle=(phase=='train'),pin_memory = (phase=='train'), drop_last=True) \\\n              for phase in ['train','valid'] }","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.860591Z","iopub.execute_input":"2022-02-05T16:12:14.860965Z","iopub.status.idle":"2022-02-05T16:12:14.872359Z","shell.execute_reply.started":"2022-02-05T16:12:14.860925Z","shell.execute_reply":"2022-02-05T16:12:14.871533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binarize(T, nb_classes):\n    T = T.cpu().numpy()\n    \n    T = sklearn.preprocessing.label_binarize(\n        T, classes = range(0, nb_classes)\n    )\n    T = torch.FloatTensor(T).to(config.device)\n    return T\n\ndef l2_norm(input):\n    input_size = input.size()\n    buffer = torch.pow(input, 2)\n    normp = torch.sum(buffer, 1).add_(1e-12)\n    norm = torch.sqrt(normp)\n    _output = torch.div(input, norm.view(-1, 1).expand_as(input))\n    output = _output.view(input_size)\n    return output\nclass Proxy_Anchor(torch.nn.Module):\n    def __init__(self, nb_classes, sz_embed, mrg = 0.1, alpha = 32):\n        torch.nn.Module.__init__(self)\n        # Proxy Anchor Initialization\n        self.proxies = torch.nn.Parameter(torch.randn(nb_classes, sz_embed).cuda())\n        nn.init.kaiming_normal_(self.proxies, mode='fan_out')\n\n        self.nb_classes = nb_classes\n        self.sz_embed = sz_embed\n        self.mrg = mrg\n        self.alpha = alpha\n        \n    def forward(self, X, T):\n        P = self.proxies\n\n        cos = F.linear(l2_norm(X), l2_norm(P))  # Calcluate cosine similarity\n        P_one_hot = binarize(T = T, nb_classes = self.nb_classes)\n        N_one_hot = 1 - P_one_hot\n    \n        pos_exp = torch.exp(-self.alpha * (cos - self.mrg))\n        neg_exp = torch.exp(self.alpha * (cos + self.mrg))\n\n        with_pos_proxies = torch.nonzero(P_one_hot.sum(dim = 0) != 0).squeeze(dim = 1)   # The set of positive proxies of data in the batch\n        num_valid_proxies = len(with_pos_proxies)   # The number of positive proxies\n        \n        P_sim_sum = torch.where(P_one_hot == 1, pos_exp, torch.zeros_like(pos_exp)).sum(dim=0) \n        N_sim_sum = torch.where(N_one_hot == 1, neg_exp, torch.zeros_like(neg_exp)).sum(dim=0)\n        \n        pos_term = torch.log(1 + P_sim_sum).sum() / num_valid_proxies\n        neg_term = torch.log(1 + N_sim_sum).sum() / self.nb_classes\n        loss = pos_term + neg_term     \n        \n        return loss\n","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.873914Z","iopub.execute_input":"2022-02-05T16:12:14.874376Z","iopub.status.idle":"2022-02-05T16:12:14.889604Z","shell.execute_reply.started":"2022-02-05T16:12:14.874335Z","shell.execute_reply":"2022-02-05T16:12:14.888746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Densenet121(nn.Module):\n    def __init__(self):\n        super(Densenet121, self).__init__()\n\n        self.backbone = models.densenet121(pretrained=True)\n        \n        in_feature = self.backbone.classifier.in_features\n        self.backbone.classifier = nn.Linear(in_feature, config.num_embeddings)\n        \n    def forward(self, x):\n        x = F.normalize(self.backbone(x))\n        return x\n\nif config.model_name == 'densenet121':\n    model = Densenet121()\n\nmodel = model.to(config.device)\nif config.loss_type == 'proxy_anchor':\n    loss_func = Proxy_Anchor(nb_classes=config.num_class, sz_embed = config.num_embeddings, mrg=0.1, alpha=32).to(config.device)\n\nparams_group = [{'params': model.parameters(), 'lr':float(config.lr)},]\nif config.loss_type == 'proxy_anchor' or config.loss_type == 'proxy_nca':\n    params_group.append({'params': loss_func.proxies, 'lr':float(config.lr)*10})\n\nif config.optimizer =='sgd':\n    optimizer = torch.optim.SGD(params_group, lr=float(config.lr), weight_decay = config.weight_decay, momentum = 0.9, nesterov=True)\nelif config.optimizer == 'adam':\n    optimizer = torch.optim.Adam(params_group, lr=float(config.lr), weight_decay = config.weight_decay)\nelif config.optimizer == 'rmsprop':\n    optimizer = torch.optim.RMSprop(params_group, lr=float(config.lr), alpha=0.9, weight_decay = config.weight_decay, momentum = 0.9)\nelif config.optimizer == 'adamw':\n    optimizer = torch.optim.AdamW(params_group, lr=float(config.lr), weight_decay = config.weight_decay)\n\nscheduler = optim.lr_scheduler.StepLR(optimizer, step_size = config.lr_decay_step, gamma=config.lr_decay_gamma)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:14.891075Z","iopub.execute_input":"2022-02-05T16:12:14.891372Z","iopub.status.idle":"2022-02-05T16:12:22.906633Z","shell.execute_reply.started":"2022-02-05T16:12:14.891333Z","shell.execute_reply":"2022-02-05T16:12:22.905811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, loss_func, train_loader, optimizer, epoch, scheduler):\n    model.train()\n    for batch_idx, (data, labels) in enumerate(train_loader):\n        data, labels = data.to(config.device), labels.to(config.device)\n        optimizer.zero_grad()\n        embeddings = model(data)\n        loss = loss_func(embeddings, labels)\n        loss.backward()\n        optimizer.step()\n        if batch_idx % 20 == 0:\n            print(\"Epoch {} Iteration {}: Loss = {}\".format(epoch, batch_idx, loss))\n    if scheduler is not None:\n        scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:22.908558Z","iopub.execute_input":"2022-02-05T16:12:22.908827Z","iopub.status.idle":"2022-02-05T16:12:22.918462Z","shell.execute_reply.started":"2022-02-05T16:12:22.908798Z","shell.execute_reply":"2022-02-05T16:12:22.91762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_recall_at_k(T, Y, k):\n    \"\"\"\n    T : [nb_samples] (target labels)\n    Y : [nb_samples x k] (k predicted labels/neighbours)\n    \"\"\"\n    s = 0\n    for t,y in zip(T,Y):\n        if t in torch.Tensor(y).long()[:k]:\n            s += 1\n    return s / (1. * len(T))\ndef l2_norm(input):\n    input_size = input.size()\n    buffer = torch.pow(input, 2)\n    normp = torch.sum(buffer, 1).add_(1e-12)\n    norm = torch.sqrt(normp)\n    _output = torch.div(input, norm.view(-1, 1).expand_as(input))\n    output = _output.view(input_size)\n\n    return output\ndef predict_batchwise(model, dataloader):\n    \n    model_is_training = model.training\n    model.eval()\n    \n    ds = dataloader.dataset\n    A = [[] for i in range(len(ds[0]))]\n    with torch.no_grad():\n        # extract batches (A becomes list of samples)\n        for batch in tqdm(dataloader):\n            for i, J in enumerate(batch):\n                # i = 0: sz_batch * images\n                # i = 1: sz_batch * labels\n                # i = 2: sz_batch * indices\n                if i == 0:\n                    # move images to device of model (approximate device)\n                    J = model(J.to(config.device))\n\n                for j in J:\n                    A[i].append(j)\n    model.train()\n    model.train(model_is_training) # revert to previous training state\n    \n    return [torch.stack(A[i]) for i in range(len(A))]\ndef evaluate_cos_SOP(model, dataloader):\n    nb_classes = config.num_class\n    \n    # calculate embeddings with model and get targets\n    X, T = predict_batchwise(model, dataloader)\n    X = l2_norm(X)\n    \n    # get predictions by assigning nearest 8 neighbors with cosine\n    K = 1000\n    Y = []\n    xs = []\n    for x in X:\n        if len(xs)<10000:\n            xs.append(x)\n        else:\n            xs.append(x)            \n            xs = torch.stack(xs,dim=0)\n            cos_sim = F.linear(xs,X)\n            y = T[cos_sim.topk(1 + K)[1][:,1:]]\n            Y.append(y.float().cpu())\n            xs = []\n            \n    # Last Loop\n    xs = torch.stack(xs,dim=0)\n    cos_sim = F.linear(xs,X)\n    y = T[cos_sim.topk(1 + K)[1][:,1:]]\n    Y.append(y.float().cpu())\n    Y = torch.cat(Y, dim=0)\n\n    # calculate recall @ 1, 2, 4, 8\n    recall = []\n    for k in [1, 2, 4, 5, 10, 100, 1000]:\n        r_at_k = calc_recall_at_k(T, Y, k)\n        recall.append(r_at_k)\n        print(\"R@{} : {:.3f}\".format(k, 100 * r_at_k))\n    return recall","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:22.920063Z","iopub.execute_input":"2022-02-05T16:12:22.920338Z","iopub.status.idle":"2022-02-05T16:12:23.014912Z","shell.execute_reply.started":"2022-02-05T16:12:22.920302Z","shell.execute_reply":"2022-02-05T16:12:23.013986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.continue_train==True:\n    print('loading status...')\n    checkpoint = torch.load(config.resume_model)\n    model.load_state_dict(checkpoint['model_state_dict'])\nf = open(config.path_save_log+\"recall.txt\", 'w', buffering=1)\nbest_recall = 0\nmodel.to(config.device)\nwriter = SummaryWriter(config.path_save_log)\nfor epoch in range(config.start_epoch, config.train_number_epochs):\n    train(model, loss_func, dataloader['train'], optimizer, epoch, scheduler)\n    \n    r_at_1 = evaluate_cos_SOP(model, dataloader['valid'])\n    print('epoch: {}, recall@1: {}'.format(epoch, r_at_1))\n    r_at_1 = r_at_1[0]\n    \n    torch.save({\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'scheduler': scheduler,\n            'epoch': epoch\n                }, config.path_save_model+ '_last.pt')\n    \n    if r_at_1 > best_recall:\n        best_recall = r_at_1\n        torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'scheduler': scheduler,\n                'epoch': epoch\n                    }, config.path_save_model+ '_best.pt')\n    f.write(str(r_at_1)+\"\\n\")\n    f.flush()\n    writer.add_scalar('precision@1', r_at_1, epoch)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T16:12:23.017638Z","iopub.execute_input":"2022-02-05T16:12:23.017903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}