{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# import module we'll need to import our custom module\nfrom shutil import copyfile\n\ncopyfile(src = \"../input/imetfgvc/ml_stratifiers.py\", dst = \"../working/ml_stratifiers.py\")\n\n# import all our functions\n\nfrom ml_stratifiers import MultilabelStratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport copy\nimport sys\nfrom PIL import Image\nimport time \nfrom tqdm.autonotebook import tqdm\nimport random\nimport gc\nimport cv2\nimport scipy\nimport math\nimport matplotlib.pyplot as plt\n\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.optim.optimizer import Optimizer\nimport torch.backends.cudnn as cudnn\nfrom torch.autograd import Variable\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR, _LRScheduler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy.special\n\nSEED = 1996\nbase_dir = '../input/'\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    os.environ['PYHTONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(SEED)\n\ndef l2_norm(input,axis=1):\n    norm = torch.norm(input,2,axis,True)\n    output = torch.div(input, norm)\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/imet-2019-fgvc6/train.csv')\nlabels_df = pd.read_csv('../input/imet-2019-fgvc6/labels.csv')\ntest_df = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\n\nmlb = MultiLabelBinarizer()\nlabels_encoded = mlb.fit_transform(train_df.attribute_ids)\n\ndef get_label(attribute_ids):\n    attribute_ids = attribute_ids.split()\n    for _,ids in enumerate(attribute_ids):\n        attribute_ids[_] = int(ids)\n    one_hot = torch.zeros(1103).scatter_(0, torch.LongTensor(attribute_ids), 1)\n    return one_hot\n\ntrain_df['attribute_ids_encoded'] = train_df['attribute_ids'].apply(get_label)\n\nimg_class_dict = {k:v for k, v in zip(train_df.id, train_df.attribute_ids_encoded)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class iMetDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', idx=[], transform = transforms.Compose([transforms.RandomResizedCrop(128),transforms.ToTensor()]), \\\n                labels_dict={}):\n\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.image_files_list = [self.image_files_list[i] for i in idx]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        img = Image.open(img_name)\n\n        image = self.transform(img)\n\n        #image = random_erase(image)\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            #label = get_label(self.labels_dict[img_name_short])\n            label = self.labels_dict[img_name_short]\n        else:\n            label = torch.zeros(1103)\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms_crop = transforms.Compose([\n    transforms.RandomApply([\n        lambda im: transforms.RandomCrop(min(im.size[0],im.size[1]))(im),\n    ], p=0.35),\n    transforms.RandomResizedCrop(352, scale=(0.6, 1.0), ratio=(0.8, 1.25)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ])\n\ndata_transforms_crop_test = transforms.Compose([\n    transforms.RandomApply([\n        lambda im: transforms.RandomCrop(min(im.size[0],im.size[1]))(im),\n    ], p=0.35),\n    transforms.RandomResizedCrop(352 ,scale=(0.6, 1.0), ratio=(0.9, 1.11)),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################################################################################################### Define Cycle LR\n# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\nclass CyclicLR(object):\n    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n                 step_size=2000, mode='triangular', gamma=1.,\n                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(\n                type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n            if len(base_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} base_lr, got {}\".format(\n                    len(optimizer.param_groups), len(base_lr)))\n            self.base_lrs = list(base_lr)\n        else:\n            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n\n        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n            if len(max_lr) != len(optimizer.param_groups):\n                raise ValueError(\"expected {} max_lr, got {}\".format(\n                    len(optimizer.param_groups), len(max_lr)))\n            self.max_lrs = list(max_lr)\n        else:\n            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n\n        self.step_size = step_size\n\n        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n                and scale_fn is None:\n            raise ValueError('mode is invalid and scale_fn is None')\n\n        self.mode = mode\n        self.gamma = gamma\n\n        if scale_fn is None:\n            if self.mode == 'triangular':\n                self.scale_fn = self._triangular_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = self._triangular2_scale_fn\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = self._exp_range_scale_fn\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n\n        self.batch_step(last_batch_iteration + 1)\n        self.last_batch_iteration = last_batch_iteration\n\n    def batch_step(self, batch_iteration=None):\n        if batch_iteration is None:\n            batch_iteration = self.last_batch_iteration + 1\n        self.last_batch_iteration = batch_iteration\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr\n\n    def _triangular_scale_fn(self, x):\n        return 1.\n\n    def _triangular2_scale_fn(self, x):\n        return 1 / (2. ** (x - 1))\n\n    def _exp_range_scale_fn(self, x):\n        return self.gamma**(x)\n\n    def get_lr(self):\n        step_size = float(self.step_size)\n        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n\n        lrs = []\n        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n        for param_group, base_lr, max_lr in param_lrs:\n            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n            if self.scale_mode == 'cycle':\n                lr = base_lr + base_height * self.scale_fn(cycle)\n            else:\n                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n            lrs.append(lr)\n        return lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.utils.weight_norm as weightNorm\nimport torch.nn.init as init\n\ndef l2_norm(input,axis=1):\n    norm = torch.norm(input,2,axis,True)\n    output = torch.div(input, norm)\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################################################################################################################### Loss\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n\n    def forward(self, logit, target):\n        target = target.float()\n        max_val = (-logit).clamp(min=0)\n        loss = logit - logit * target + max_val + \\\n               ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n        invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n        loss = (invprobs * self.gamma).exp() * loss\n        if len(loss.size())==2:\n            loss = loss.sum(dim=1)\n        return loss.mean()\n\nclass FbetaLoss(nn.Module):\n    def __init__(self, beta=1):\n        super(FbetaLoss, self).__init__()\n        self.small_value = 1e-6\n        self.beta = beta\n\n    def forward(self, logits, labels):\n        beta = self.beta\n        batch_size = logits.size()[0]\n        p = F.sigmoid(logits)\n        l = labels\n        num_pos = torch.sum(p, 1) + self.small_value\n        num_pos_hat = torch.sum(l, 1) + self.small_value\n        tp = torch.sum(l * p, 1)\n        precise = tp / num_pos\n        recall = tp / num_pos_hat\n        fs = (1 + beta * beta) * precise * recall / (beta * beta * precise + recall + self.small_value)\n        loss = fs.sum() / batch_size\n        return 1 - loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function, division\n\nimport torch\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\nimport numpy as np\ntry:\n    from itertools import  ifilterfalse\nexcept ImportError: # py3k\n    from itertools import  filterfalse\n\n\ndef lovasz_grad(gt_sorted):\n    \"\"\"\n    Computes gradient of the Lovasz extension w.r.t sorted errors\n    See Alg. 1 in paper\n    \"\"\"\n    p = len(gt_sorted)\n    gts = gt_sorted.sum()\n    intersection = gts - gt_sorted.float().cumsum(0)\n    union = gts + (1 - gt_sorted).float().cumsum(0)\n    jaccard = 1. - intersection / union\n    if p > 1: # cover 1-pixel case\n        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]\n    return jaccard\n\n\ndef iou_binary(preds, labels, EMPTY=1., ignore=None, per_image=True):\n    \"\"\"\n    IoU for foreground class\n    binary: 1 foreground, 0 background\n    \"\"\"\n    if not per_image:\n        preds, labels = (preds,), (labels,)\n    ious = []\n    for pred, label in zip(preds, labels):\n        intersection = ((label == 1) & (pred == 1)).sum()\n        union = ((label == 1) | ((pred == 1) & (label != ignore))).sum()\n        if not union:\n            iou = EMPTY\n        else:\n            iou = float(intersection) / union\n        ious.append(iou)\n    iou = mean(ious)    # mean accross images if per_image\n    return 100 * iou\n\n\ndef iou(preds, labels, C, EMPTY=1., ignore=None, per_image=False):\n    \"\"\"\n    Array of IoU for each (non ignored) class\n    \"\"\"\n    if not per_image:\n        preds, labels = (preds,), (labels,)\n    ious = []\n    for pred, label in zip(preds, labels):\n        iou = []    \n        for i in range(C):\n            if i != ignore: # The ignored label is sometimes among predicted classes (ENet - CityScapes)\n                intersection = ((label == i) & (pred == i)).sum()\n                union = ((label == i) | ((pred == i) & (label != ignore))).sum()\n                if not union:\n                    iou.append(EMPTY)\n                else:\n                    iou.append(float(intersection) / union)\n        ious.append(iou)\n    ious = map(mean, zip(*ious)) # mean accross images if per_image\n    return 100 * np.array(ious)\n\n\n# --------------------------- BINARY LOSSES ---------------------------\n\n\ndef lovasz_hinge(logits, labels, per_image=True, ignore=None):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n      per_image: compute the loss per image instead of per batch\n      ignore: void class id\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_hinge_flat(*flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))\n                          for log, lab in zip(logits, labels))\n    else:\n        loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))\n    return loss\n\n\ndef lovasz_hinge_flat(logits, labels):\n    \"\"\"\n    Binary Lovasz hinge loss\n      logits: [P] Variable, logits at each prediction (between -\\infty and +\\infty)\n      labels: [P] Tensor, binary ground truth labels (0 or 1)\n      ignore: label to ignore\n    \"\"\"\n    if len(labels) == 0:\n        # only void pixels, the gradients should be 0\n        return logits.sum() * 0.\n    signs = 2. * labels.float() - 1.\n    errors = (1. - logits * signs)\n    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)\n    perm = perm.data\n    gt_sorted = labels[perm]\n    grad = lovasz_grad(gt_sorted)\n    # loss = torch.dot(F.relu(errors_sorted), grad)\n    # print('elu!!!!!!')\n    loss = torch.dot(F.elu(errors_sorted)+1, grad)\n    # loss = torch.dot(F.leaky_relu(errors_sorted)+1, grad)\n    return loss\n\n\ndef flatten_binary_scores(scores, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch (binary case)\n    Remove labels equal to 'ignore'\n    \"\"\"\n    scores = scores.view(-1)\n    labels = labels.view(-1)\n    if ignore is None:\n        return scores, labels\n    valid = (labels != ignore)\n    vscores = scores[valid]\n    vlabels = labels[valid]\n    return vscores, vlabels\n\n\nclass StableBCELoss(torch.nn.modules.Module):\n    def __init__(self):\n        super(StableBCELoss, self).__init__()\n    def forward(self, input, target):\n        neg_abs = - input.abs()\n        loss = input.clamp(min=0) - input * target + (1 + neg_abs.exp()).log()\n        return loss.mean()\n\n\ndef binary_xloss(logits, labels, ignore=None):\n    \"\"\"\n    Binary Cross entropy loss\n      logits: [B, H, W] Variable, logits at each pixel (between -\\infty and +\\infty)\n      labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)\n      ignore: void class id\n    \"\"\"\n    logits, labels = flatten_binary_scores(logits, labels, ignore)\n    loss = StableBCELoss()(logits, Variable(labels.float()))\n    return loss\n\n\n# --------------------------- MULTICLASS LOSSES ---------------------------\n\n\ndef lovasz_softmax(probas, labels, only_present=False, per_image=False, ignore=None):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [B, C, H, W] Variable, class probabilities at each prediction (between 0 and 1)\n      labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1)\n      only_present: average only on classes present in ground truth\n      per_image: compute the loss per image instead of per batch\n      ignore: void class labels\n    \"\"\"\n    if per_image:\n        loss = mean(lovasz_softmax_flat(*flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), only_present=only_present)\n                          for prob, lab in zip(probas, labels))\n    else:\n        loss = lovasz_softmax_flat(*flatten_probas(probas, labels, ignore), only_present=only_present)\n    return loss\n\n\ndef lovasz_softmax_flat(probas, labels, only_present=False):\n    \"\"\"\n    Multi-class Lovasz-Softmax loss\n      probas: [P, C] Variable, class probabilities at each prediction (between 0 and 1)\n      labels: [P] Tensor, ground truth labels (between 0 and C - 1)\n      only_present: average only on classes present in ground truth\n    \"\"\"\n    C = probas.size(1)\n    losses = []\n    for c in range(C):\n        fg = (labels == c).float() # foreground for class c\n        if only_present and fg.sum() == 0:\n            continue\n        errors = (Variable(fg) - probas[:, c]).abs()\n        errors_sorted, perm = torch.sort(errors, 0, descending=True)\n        perm = perm.data\n        fg_sorted = fg[perm]\n        losses.append(torch.dot(errors_sorted, Variable(lovasz_grad(fg_sorted))))\n    return mean(losses)\n\n\ndef flatten_probas(probas, labels, ignore=None):\n    \"\"\"\n    Flattens predictions in the batch\n    \"\"\"\n    B, C, H, W = probas.size()\n    probas = probas.permute(0, 2, 3, 1).contiguous().view(-1, C)  # B * H * W, C = P, C\n    labels = labels.view(-1)\n    if ignore is None:\n        return probas, labels\n    valid = (labels != ignore)\n    vprobas = probas[valid.nonzero().squeeze()]\n    vlabels = labels[valid]\n    return vprobas, vlabels\n\ndef xloss(logits, labels, ignore=None):\n    \"\"\"\n    Cross entropy loss\n    \"\"\"\n    return F.cross_entropy(logits, Variable(labels), ignore_index=255)\n\n\n# --------------------------- HELPER FUNCTIONS ---------------------------\n\ndef mean(l, ignore_nan=False, empty=0):\n    \"\"\"\n    nanmean compatible with generators.\n    \"\"\"\n    l = iter(l)\n    if ignore_nan:\n        l = ifilterfalse(np.isnan, l)\n    try:\n        n = 1\n        acc = next(l)\n    except StopIteration:\n        if empty == 'raise':\n            raise ValueError('Empty mean')\n        return empty\n    for n, v in enumerate(l, 2):\n        acc += v\n    if n == 1:\n        return acc\n    return acc / n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CombineLoss(nn.Module):\n    def __init__(self):\n        super(CombineLoss, self).__init__()\n        self.fbeta_loss = FbetaLoss(beta=2)\n        self.focal_loss = FocalLoss()\n        self.bce_loss = nn.BCEWithLogitsLoss()\n        \n    def forward(self, logits, labels):\n        loss_fbeta_loss = self.fbeta_loss(logits, labels)\n        loss_focal = self.focal_loss(logits, labels)\n        loss_lovasz = lovasz_hinge(logits, labels)\n        return 1.2*loss_lovasz + loss_focal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorchcv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pytorchcv.model_provider import get_model as ptcv_get_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class swish(nn.Module):\n    def __init__(self):\n        super(swish, self).__init__()\n        self.sigmoid = nn.Sigmoid()\n    def forward(self, x):\n        return x*self.sigmoid(x)\n\nmodel_conv = ptcv_get_model(\"ibn_resnext101_32x4d\", pretrained=True)\nmodel_conv.features.final_pool = nn.AdaptiveAvgPool2d(1)\nmodel_conv.output = nn.Linear(in_features=2048, out_features=1103, bias=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################################################################################################# Define training\nfrom torch.nn.parallel.data_parallel import data_parallel\n\nnum_classes = 1103\n\nmodel_conv.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = CombineLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nval_batch_size = 128\nnum_workers = 4\nnum_epoch = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splts = 7\nmskf = MultilabelStratifiedKFold(n_splits=n_splts, random_state=SEED)\nsplits = mskf.split(train_df['id'], labels_encoded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######################################### Define find threshold function\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))\n\ndef threshold_search(y_pred, y_true):\n    score = []\n    candidates = np.arange(0.2, 0.5, 0.01)\n    for th in candidates:\n        yp = (y_pred > th*np.ones_like(y_pred)).astype(int)\n        #print(yp)\n        #print(y_true)\n        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average=\"samples\"))\n    score = np.array(score)\n    pm = score.argmax()\n    best_th, best_score = candidates[pm], score[pm]\n\n\n    return best_th, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def PairwiseConfusion(features, target):\n    features = nn.Sigmoid()(features)\n    batch_size = features.size(0)\n    if float(batch_size) % 2 != 0:\n        batch_left = features[:int(0.5*(batch_size-1))]\n        batch_right = features[int(0.5*(batch_size-1)):batch_size-1]\n        target_left = target[:int(0.5*(batch_size-1))]\n        target_right = target[int(0.5*(batch_size-1)):batch_size-1]\n\n    else:\n        batch_left = features[:int(0.5*batch_size)]\n        batch_right = features[int(0.5*batch_size):]\n        target_left = target[:int(0.5*batch_size)]\n        target_right = target[int(0.5*batch_size):]\n\n    #target_mask_t = torch.eq(target_left, target_right) # get (batchsize/2, target dim) all 1 tensor is equal \n    #target_mask_tensor_s = torch.sum(1 - target_mask_t, 1) # get (batchsize/2,) non 0 is non-equal, 0 is equal\n    #target_mask_tensor_n = torch.eq(torch.zeros_like(target_mask_tensor_s), target_mask_tensor_s) # get (batchsize/2,) 0 is non-equal, 1 is equal\n    #target_mask_tensor = 1 - target_mask_tensor_n # get (batchsize/2,) 1 is non-equal, 0 is equal\n\n    #target_mask_tensor = target_mask_tensor.type(torch.cuda.FloatTensor)\n\n    #number = target_mask_tensor.sum()\n\n    loss  = torch.norm((batch_left - batch_right).abs(),2, 1).sum() / float(batch_size)\n    #loss = torch.norm((batch_left - batch_right).abs(),2, 1)*target_mask_tensor / float(number)\n\n    return loss\n\ndef EntropicConfusion(features):\n    features = nn.Sigmoid()(features)\n    batch_size = features.size(0)\n    return torch.mul(features, torch.log(features)).sum() * (1.0 / batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fix_fold = 0 #just run this fold\nfor fold_, (tr, val) in enumerate(splits):\n    if(fold_<fix_fold):\n        continue\n    if(fold_>fix_fold):\n        break\n        \n    ################################### split data\n\n    dataset = iMetDataset(datafolder='../input/imet-2019-fgvc6/train/', datatype='train', idx=tr, \\\n            transform=data_transforms_crop, labels_dict=img_class_dict)\n    val_set = iMetDataset(datafolder='../input/imet-2019-fgvc6/train/', datatype='train', idx=val,\\\n        transform=data_transforms_crop_test, labels_dict=img_class_dict)\n    \n    #model_conv.load_state_dict(torch.load(\"../input/pytorch-model-zoo/densenet121-fbdb23505.pth\"), strict=False)\n    \n    ##################################\n    print(time.ctime(), 'Fold:', fold_+1)\n    \n    valid_f2_max = -np.Inf\n    # current number of tests, where validation f2 didn't increase\n    p_max = 0\n    lr_p_max = 0\n    patience = 5\n    lr_patience = 1\n    # whether training should be stopped\n    stop = False\n    start = 0\n    step_size = 8000\n    \n    lr = 1.5e-2\n    optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model_conv.parameters()),\\\n                                   lr, weight_decay=0.0002, momentum=0.9)\n\n    base_lr, max_lr = lr/6, lr \n    scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr, step_size=step_size,\n                mode='exp_range', gamma=0.99994)\n    \n    valid_loader = torch.utils.data.DataLoader(val_set, batch_size=val_batch_size, num_workers=num_workers)\n    ####################################### Training\n    for epoch in range(start, num_epoch): \n\n        if(epoch+1==6):\n            lr /= 4\n            print(\"lr changing from \", lr*4, \" to \", lr)\n            for g in optimizer.param_groups:\n                g['lr'] = lr\n            \n            base_lr, max_lr = lr/6, lr \n            scheduler.base_lrs = [base_lr]\n            scheduler.max_lrs = [max_lr]\n        \n        print(\"current lr is: \", lr)\n        if(stop):\n            break\n        \n        seed_everything(SEED+epoch)\n        train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n        \n        print(time.ctime(), 'Epoch:', epoch+1)\n            \n        train_loss = []\n        train_f2 = []\n\n        model_conv.train()\n            \n        for tr_batch_i, (data, target) in enumerate(train_loader):     \n\n            scheduler.batch_step()\n\n            data, target = data.cuda(), target.cuda()\n            optimizer.zero_grad()\n\n            results = data_parallel(model_conv, data)\n\n            loss = criterion(results, target.float())\n\n            loss.backward()\n\n            torch.nn.utils.clip_grad_norm_(model_conv.parameters(), max_norm=5.0, norm_type=2)\n            optimizer.step()\n\n            train_loss.append(loss.item()) \n\n            a = target.data.detach().cpu().numpy()\n            b = results.detach().cpu().numpy()\n\n            eval_step = len(train_loader)   \n\n            if (tr_batch_i+1)%eval_step == 0:  \n\n                model_conv.eval()\n                val_loss = []\n                val_f2 = []\n                y_pred_val = np.zeros((len(val), 1103))\n                y_true_val = np.zeros((len(val), 1103))\n\n                with torch.no_grad():\n                    for val_batch_i, (data, target) in enumerate(valid_loader):\n                        data, target = data.cuda(), target.cuda()\n\n                        results = data_parallel(model_conv, data)\n\n\n                        loss = criterion(results, target.float())\n                        val_loss.append(loss.item()) \n\n\n                        a = target.data.detach().cpu().numpy()\n                        b = results.detach().cpu().numpy()\n\n                        y_pred_val[val_batch_i*val_batch_size:val_batch_i*val_batch_size+b.shape[0]] = sigmoid(b)\n                        y_true_val[val_batch_i*val_batch_size:val_batch_i*val_batch_size+b.shape[0]] = a\n\n                best_threshold_val, best_score_val = threshold_search(y_pred_val, y_true_val)\n                val_f2.append(best_score_val)\n\n\n                print('Epoch %d, batches:%d, train loss: %.4f, valid loss: %.4f.'%(epoch+1, tr_batch_i, np.mean(train_loss), np.mean(val_loss)))\n\n                print('Epoch %d, batches:%d, valid f2: %.4f.'%(epoch+1, tr_batch_i, np.mean(val_f2)))\n\n                valid_f2 = np.mean(val_f2)\n                if valid_f2 > valid_f2_max:\n\n                    print('Validation f2 increased ({:.6f} --> {:.6f}).  Saving model ...'.format(\\\n                    valid_f2_max, valid_f2))\n\n                    torch.save(model_conv.state_dict(), \"model_\"+str(fold_))\n                    valid_f2_max = valid_f2\n                    p_max = 0\n                    lr_p_max = 0\n\n                else:\n                    print(\"Validatione f2 doesn't increase\")\n                    p_max += 1\n                    lr_p_max += 1\n                \n                if p_max > patience:\n                    stop = True\n                    break \n\n                if(lr_p_max>=lr_patience):\n                    print(\"lr change from: \", lr, \" to \", lr/4)\n                    lr /= 4\n                    for g in optimizer.param_groups:\n                        g['lr'] = lr\n\n                    base_lr, max_lr = lr/6, lr \n                    scheduler.base_lrs = [base_lr]\n                    scheduler.max_lrs = [max_lr]\n\n                    lr_p_max = 0\n                torch.cuda.empty_cache() \n                model_conv.train()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}