{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🐬 Happy Whale V6 ","metadata":{}},{"cell_type":"markdown","source":"### 🐋 Trick List","metadata":{}},{"cell_type":"markdown","source":"* **GeM**\n* **ArcFace**\n* **Label Smoothing**\n* **Focal Loss**\n* **CutMix and MixUp**\n* **TTA**","metadata":{}},{"cell_type":"markdown","source":"### 🐋 In Happy Whale V6","metadata":{}},{"cell_type":"markdown","source":"* **Detic Crop**\n* **KFolds Stratified**\n* **U2Net**\n* **EMA**","metadata":{}},{"cell_type":"markdown","source":"### 🐋 In Happy Whale V7","metadata":{}},{"cell_type":"markdown","source":"* **Focus loss to Asymmetric Loss**\n* **Metric Loss with Margin**\n* **AdamW to AdamP**","metadata":{}},{"cell_type":"markdown","source":"### 🐋 Log","metadata":{}},{"cell_type":"markdown","source":"**2022/3/25**","metadata":{}},{"cell_type":"markdown","source":"* model = SegConvWhaleNet\n* backbone = 'convnext_small'\n* lr = 3e-4\n* img_size = 384\n* batch_size = 32\n* weight_decay = 2e-5\n* Epochs = 25","metadata":{}},{"cell_type":"markdown","source":"**2022/3/22**\n","metadata":{}},{"cell_type":"markdown","source":"I noticed that the original minimum lr was too small (about 1e-9), so I changed it to 1.5e-6.","metadata":{}},{"cell_type":"code","source":"! pip install timm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/NathanUA/U-2-Net.git","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import notebook\nfrom IPython import display\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F \nfrom torch.utils.data import Dataset\nfrom torchvision.transforms import ToTensor\nfrom torch.optim.lr_scheduler import OneCycleLR\nfrom sklearn.model_selection import StratifiedKFold\nsys.path.append('./U-2-Net')\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nfrom timm.optim.optim_factory import create_optimizer\nfrom model import U2NET","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Print Style**","metadata":{}},{"cell_type":"code","source":"class clr:\n    S = '\\033[1m' + '\\033[96m'\n    E = '\\033[0m'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Set Seeds**","metadata":{}},{"cell_type":"code","source":"def set_seeds(seed = 2022 + 3 + 22):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed) \n    torch.backends.cudnn.benchmark = True\n    os.environ['PYTHONHASHSEED'] = str(seed) # Set a fixed value for the hash seed\n    \nset_seeds()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Config**","metadata":{}},{"cell_type":"code","source":"cfg = { 'Effnet_B0': 'tf_efficientnet_b0',\n        'Effnet_B4': 'tf_efficientnet_b4',\n       'EffnetV2_s': 'tf_efficientnetv2_s_in21k',\n       'EffnetV2_m': 'tf_efficientnetv2_m_in21k',\n       'ConvNext_s': 'convnext_small',\n       'ConvNext_b': 'convnext_base_384_in22ft1k',\n      }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR = 3e-4\nimg_size = 56 #384\nBATCH_SIZE = 64\nNUM_WORKERS = 2\nNUM_CLASSES = 15587","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_IMG_DIR = r'../input/happy-whale-and-dolphin/train_images'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read CSV and Make Labels**","metadata":{}},{"cell_type":"code","source":"# train_df_path = r'../input/whale2-cropped-dataset/train2.csv'\n# train_df = pd.read_csv(train_df_path)\n# train_df['label'] = train_df.groupby('individual_id').ngroup()\n# train_df['sp_label'] = train_df.groupby('species').ngroup()\n# train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Stratified KFold**","metadata":{}},{"cell_type":"code","source":"# N_SPLITS = 10\n\n# skfolds = StratifiedKFold(n_splits=N_SPLITS)\n\n# for n_fold, (train_data, valid_data) in enumerate(skfolds.split(X=train_df, y=train_df['individual_id'])):\n# train_df.loc[train_data, 'kfold:'+str(n_fold)] = 'train'\n# train_df.loc[valid_data, 'kfold:'+str(n_fold)] = 'valid'\n    \n# train_df.to_csv('train.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Dataset","metadata":{}},{"cell_type":"code","source":"class HappyWhaleDataset(Dataset):\n    def __init__(self, df:pd.DataFrame, img_dir:str, transform=None, crop_p=0.6):\n        self.df = df\n        self.img_dir = img_dir\n        self.transform = transform\n        self.crop_p = crop_p\n        \n        \n    def __getitem__(self, idx):\n        img_name = self.df['image'][idx]\n        #label = self.df['label'][idx]\n        label = self.df['sp_label'][idx]\n        \n        img_path = os.path.join(self.img_dir, img_name)\n        image = Image.open(img_path).convert('RGB')\n        image = np.array(image)\n        \n        if self.crop_p > 0 and random.uniform(0,1) < self.crop_p:\n            if self.df['box'][idx] != 'nan':\n                xmin, ymin, xmax, ymax = self.df['box'][idx]\n                image = image[ymin:ymax, xmin:xmax, :]\n        \n        if self.transform is not None:\n            image = self.transform(image=image)['image']\n\n        image = torch.tensor(image.transpose(2, 0, 1))\n        label = torch.tensor(label)\n        \n        return image, label\n        \n    def __len__(self):\n        return len(self.df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Data Augmentation","metadata":{}},{"cell_type":"markdown","source":"**1.**","metadata":{}},{"cell_type":"code","source":"from albumentations import Compose,OneOf,SmallestMaxSize, Resize,Normalize,HorizontalFlip,ShiftScaleRotate,CLAHE,RandomBrightnessContrast,Emboss,HueSaturationValue,RandomResizedCrop,MotionBlur,MedianBlur,GaussianBlur,GaussNoise,CoarseDropout,Sharpen","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = {\n    'train': Compose([\n        SmallestMaxSize(max_size=img_size),\n        RandomResizedCrop(img_size, img_size, scale=(0.75, 1.0)),\n        RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.5),\n        OneOf([Sharpen(), Emboss(), CLAHE()], p=0.5),\n        HueSaturationValue(),\n        OneOf([MotionBlur(), MedianBlur(), GaussianBlur()], p=0.5),\n        GaussNoise(),\n        HorizontalFlip(p=0.5),\n        ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=30, p=0.5),\n        CoarseDropout(max_holes=2, max_height=(img_size//10), max_width=(img_size//10), min_holes=1, min_height=(img_size//20), min_width=(img_size//20), p=0.3),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]),\n        \n    'valid': Compose([\n        SmallestMaxSize(max_size=img_size),\n        Resize(img_size, img_size),\n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]),\n    \n    'TTA': Compose([\n        SmallestMaxSize(max_size=img_size),\n        RandomResizedCrop(img_size, img_size, scale=(0.8, 1.0)),\n        HorizontalFlip(p=0.5),\n        ShiftScaleRotate(rotate_limit=15, p=0.4),                                                                \n        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ]), \n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.MixUp**","metadata":{}},{"cell_type":"code","source":"def mixup_data(input, label, alpha=0.2):\n    p = np.random.beta(alpha, alpha)\n    batch_size = input.size()[0]\n    index = torch.randperm(batch_size).to(device)\n    lam = np.maximum(p, 1-p)\n    mixed_input = lam * input + (1 - lam) * input[index, :]\n    \n    return mixed_input, label, label[index], lam","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.CutMix**","metadata":{}},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n    \n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cutmix_data(input, label, beta=1.0):\n    lam = np.random.beta(beta, beta)\n    batch_size = input.size()[0]\n    rand_index = torch.randperm(batch_size).to(device)\n    \n    label_a = label\n    label_b = label[rand_index, :]\n    \n    bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam)\n    input[:, :, bbx1:bbx2, bby1:bby2] = input[rand_index, :, bbx1:bbx2, bby1:bby2]\n    \n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n    label = lam * label_a + (1 - lam) * label_b\n    \n    return input, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Model","metadata":{}},{"cell_type":"markdown","source":"**GeM Pooling**","metadata":{}},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = nn.Parameter(torch.tensor(p, dtype=torch.float))\n        self.eps = eps\n\n    def forward(self, inputs):\n        KH = inputs.shape[-2]\n        KW = inputs.shape[-1]\n        inputs = inputs.clamp(min=self.eps).pow(self.p)\n        outputs = F.avg_pool2d(inputs, (KH, KW)).pow(1./self.p)\n        \n        return outputs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Cos Product**","metadata":{}},{"cell_type":"code","source":"class CosProduct(nn.Module):\n    def __init__(self, embedding_size, num_classes):\n        super(CosProduct, self).__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(num_classes, embedding_size))\n        nn.init.xavier_uniform_(self.weight)\n        \n    def forward(self, x):\n        x = F.normalize(x, dim=1)\n        w = F.normalize(self.weight, dim=1)\n        cosine = F.linear(x, w)\n        \n        return cosine","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1.Efficientnet backbone**","metadata":{}},{"cell_type":"code","source":"class EffWhaleNet(nn.Module):\n    def __init__(self, backbone, num_classes, embedding_size=512, backbone_pretrained=True):\n        super(EffWhaleNet, self).__init__()\n        self.pretrained = backbone_pretrained\n        self.backbone = timm.create_model(backbone, pretrained=self.pretrained)\n        backbone_features = self.backbone.classifier.in_features\n        self.backbone.global_pool = nn.Identity()\n        self.backbone.classifier = nn.Identity()\n        self.GeM = GeM()\n        self.Embedding = nn.Linear(backbone_features, embedding_size)\n        self.CosProduct = CosProduct(embedding_size, num_classes)\n        \n    def forward(self, x, only_feature=False):\n        x = self.backbone(x)\n        x = self.GeM(x).flatten(1)\n        feature = self.Embedding(x)\n        outputs = self.CosProduct(feature)\n        if only_feature:\n            return feature\n        else:\n            return outputs, feature","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.EffnetV2 backbone + U2Net (input channels = 4)**","metadata":{}},{"cell_type":"code","source":"class SegEffWhaleNet(nn.Module):\n    def __init__(self, backbone, num_classes, embedding_size=512, unet_path=None, backbone_pretrained=True):\n        super(SegEffWhaleNet, self).__init__()\n        self.THRESHOLD = 0.3\n        self.Unet = U2NET()\n        self.Unet.load_state_dict(torch.load(unet_path))\n            \n        for param in self.Unet.parameters():\n            param.requires_grad = False \n            \n        self.backbone = timm.create_model(backbone, pretrained=backbone_pretrained)\n        backbone_features = self.backbone.classifier.in_features\n        self.backbone.global_pool = nn.Identity()\n        self.backbone.classifier = nn.Identity()\n        self.GeM = GeM()\n        self.Embedding = nn.Sequential(nn.Linear(backbone_features, embedding_size), nn.Dropout(p=0.2))\n        self.FeatMerge = nn.Linear(embedding_size*2, embedding_size)\n        self.CosProduct = CosProduct(embedding_size, num_classes)\n        \n    \n    def forward(self, x):\n        pred = self.unet_pred(x)\n        pred = self.normPRED(pred)\n        mask = torch.where(pred > self.THRESHOLD, 1, 0)\n        s = x * mask\n        \n        x = self.backbone(x)\n        x = self.GeM(x).flatten(1)\n        feat_x = self.Embedding(x)\n        \n        s = self.backbone(s)\n        s = self.GeM(s).flatten(1)\n        feat_s = self.Embedding(s)\n        \n        feature = torch.cat([feat_x, feat_s], dim=1)\n        feature = self.FeatMerge(feature)\n        \n        outputs = self.CosProduct(feature)\n        \n        \n        return outputs, feature\n    \n    \n    def unet_pred(self, x):\n        self.Unet.eval()\n        with torch.no_grad():\n            d1,d2,d3,d4,d5,d6,d7 = self.Unet(x)\n            pred = d1.detach()\n            \n            del d1,d2,d3,d4,d5,d6,d7\n            \n        return pred\n    \n    \n    def normPRED(self, d):\n        batch_size = d.size()[0]\n        \n        dmin = d.reshape(batch_size, -1).min(dim=1, keepdim=True).values\n        dmin = dmin.reshape(batch_size, 1, 1, 1)\n        \n        dmax = d.reshape(batch_size, -1).max(dim=1, keepdim=True).values\n        dmax = dmax.reshape(batch_size, 1, 1, 1)\n        \n        return (d - dmin) / (dmax - dmin)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.Convnext backbone + U2Net**","metadata":{}},{"cell_type":"code","source":"class SegConvWhaleNet(nn.Module):\n    def __init__(self, backbone, num_classes, embedding_size=512, unet_path=None, backbone_pretrained=True):\n        super(SegConvWhaleNet, self).__init__()\n        self.THRESHOLD = 0.3\n        self.Unet = U2NET()\n        \n        if unet_path is not None:\n            self.Unet.load_state_dict(torch.load(unet_path))\n            \n        for param in self.Unet.parameters():\n            param.requires_grad = False \n            \n        self.backbone = timm.create_model(backbone, pretrained=backbone_pretrained)\n        backbone_features = self.backbone.get_classifier().in_features\n        self.backbone.head.global_pool = GeM()\n        self.backbone.head.fc = nn.Identity()\n        self.Embedding = nn.Linear(backbone_features, embedding_size)\n        self.Feat_Merge = nn.Linear(embedding_size * 2, embedding_size)\n        self.CosProduct = CosProduct(embedding_size, num_classes)\n        \n    \n    def forward(self, x):\n        pred = self.unet_pred(x)\n        pred = self.normPRED(pred)\n        mask = torch.where(pred > self.THRESHOLD, 1, 0)\n        s = x * mask\n        \n        x = self.backbone(x)\n        feat_x = self.Embedding(x)\n        \n        s = self.backbone(s)\n        feat_s = self.Embedding(s)\n        \n        feature = torch.cat([feat_x, feat_s], dim=1)\n        feature = self.Feat_Merge(feature)\n        \n        outputs = self.CosProduct(feature)\n        \n        return outputs, feature\n    \n    \n    def unet_pred(self, x):\n        self.Unet.eval()\n        with torch.no_grad():\n            d1,d2,d3,d4,d5,d6,d7 = self.Unet(x)\n            pred = d1.detach()\n            \n            del d1,d2,d3,d4,d5,d6,d7\n            \n        return pred\n    \n    \n    def normPRED(self, d):\n        batch_size = d.size()[0]\n        \n        dmin = d.reshape(batch_size, -1).min(dim=1, keepdim=True).values\n        dmin = dmin.reshape(batch_size, 1, 1, 1)\n        \n        dmax = d.reshape(batch_size, -1).max(dim=1, keepdim=True).values\n        dmax = dmax.reshape(batch_size, 1, 1, 1)\n        \n        return (d - dmin) / (dmax - dmin)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SegConvWhaleNetV2(nn.Module):\n    def __init__(self, backbone, num_classes, embedding_size=512, unet_path=None, backbone_pretrained=True):\n        super(SegConvWhaleNetV2, self).__init__()\n        self.THRESHOLD = 0.3\n        self.Unet = U2NET()\n        \n        if unet_path is not None:\n            self.Unet.load_state_dict(torch.load(unet_path))\n            \n        for param in self.Unet.parameters():\n            param.requires_grad = False \n            \n        self.backbone = timm.create_model(backbone, pretrained=backbone_pretrained)\n        backbone_features = self.backbone.get_classifier().in_features\n        self.backbone.head.global_pool = GeM()\n        self.backbone.head.fc = nn.Identity()\n        self.Embedding = nn.Sequential(nn.Linear(backbone_features, embedding_size), nn.Dropout(p=0.2))\n        self.FeatMerge = nn.Linear(embedding_size * 2, embedding_size)\n        self.CosProduct = CosProduct(embedding_size, num_classes)\n        \n    \n    def forward(self, x):\n        pred = self.unet_pred(x)\n        pred = self.normPRED(pred)\n        mask = torch.where(pred > self.THRESHOLD, 1, 0)\n        s = x * mask\n        \n        x = self.backbone(x)\n        feat_x = self.Embedding(x)\n        \n        s = self.backbone(s)\n        feat_s = self.Embedding(s)\n        \n        feature = torch.cat([feat_x, feat_s], dim=1)\n        feature = self.Feat_Merge(feature)\n        \n        outputs = self.CosProduct(feature)\n        \n        return outputs, feature\n    \n    \n    def unet_pred(self, x):\n        self.Unet.eval()\n        with torch.no_grad():\n            d1,d2,d3,d4,d5,d6,d7 = self.Unet(x)\n            pred = d1.detach()\n            \n            del d1,d2,d3,d4,d5,d6,d7\n            \n        return pred\n    \n    \n    def normPRED(self, d):\n        batch_size = d.size()[0]\n        \n        dmin = d.reshape(batch_size, -1).min(dim=1, keepdim=True).values\n        dmin = dmin.reshape(batch_size, 1, 1, 1)\n        \n        dmax = d.reshape(batch_size, -1).max(dim=1, keepdim=True).values\n        dmax = dmax.reshape(batch_size, 1, 1, 1)\n        \n        return (d - dmin) / (dmax - dmin)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 EMA","metadata":{}},{"cell_type":"markdown","source":"**EMA inside timm** https://fastai.github.io/timmdocs/training_modelEMA#Internals-of-Model-EMA-inside-timm","metadata":{}},{"cell_type":"code","source":"class ModelEmaV2(nn.Module):\n    def __init__(self, model, decay=0.9999, device=None):\n        super(ModelEmaV2, self).__init__()\n        self.module = deepcopy(model)\n        self.module.eval()\n        self.decay = decay\n        self.device = device\n        if self.device is not None:\n            self.module.to(device=device)\n\n    def _update(self, model, update_fn):\n        with torch.no_grad():\n            for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()):\n                if self.device is not None:\n                    model_v = model_v.to(device=self.device)\n                ema_v.copy_(update_fn(ema_v, model_v))\n\n    def update(self, model):\n        self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m)\n\n    def set(self, model):\n        self._update(model, update_fn=lambda e, m: m)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Loss Functon","metadata":{}},{"cell_type":"markdown","source":"**1.CrossEntropyLoss with Label Smoothing**","metadata":{}},{"cell_type":"code","source":"class labelsmoothing_CELoss(nn.Module):\n    def __init__(self, label_smoothing=0.1, reduction='none'):\n        super(labelsmoothing_CELoss, self).__init__()\n        assert 0 <= label_smoothing < 1\n        self.label_smoothing = label_smoothing\n        self.reduction = reduction\n        \n    def forward(self, x, label):\n        num_classes = x.shape[-1]\n        logprobs = F.log_softmax(x, dim=1)\n        label = F.one_hot(label, num_classes)\n        label = label * (1 - self.label_smoothing) + self.label_smoothing / num_classes\n        if self.reduction is 'none':\n            return (-1 * label * logprobs).sum(dim=1).reshape(-1)\n        \n        if self.reduction is 'mean':\n            return (-1 * label * logprobs).sum(dim=1).reshape(-1).mean()\n        \n        if self.reduction is 'sum':\n            return (-1 * label * logprobs).sum(dim=1).reshape(-1).sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.ArcLoss**","metadata":{}},{"cell_type":"markdown","source":"ArcFace https://arxiv.org/pdf/1801.07698v1.pdf","metadata":{}},{"cell_type":"code","source":"class ArcLoss(nn.Module):\n    def __init__(self, m=0.5, s=30, easy_margin=False):\n        super(ArcLoss, self).__init__()\n        self.m = m\n        self.s = s\n        self.cos_m = math.cos(self.m)\n        self.sin_m = math.sin(self.m)\n        self.easy_margin = easy_margin\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n        \n        \n    def forward(self, cosine, labels):\n        num_classes = cosine.size()[-1]\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m \n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        \n        one_hot = F.one_hot(labels, num_classes) \n        outputs = (one_hot * phi + (1.0 - one_hot) * cosine) * self.s\n        \n        loss = labelsmoothing_CELoss()(outputs, labels)\n        \n        return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.Focal Loss**","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha = 0.25, gamma = 2.0, reduction='none', label_smoothing=0.1):\n        super(FocalLoss, self).__init__()\n        self.s = 20\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n        self.label_smoothing = label_smoothing\n        \n    def forward(self, x, label):\n        num_classes = x.size()[-1]\n        x_s = x * self.s\n        probs = F.softmax(x_s, dim=1)\n        logprobs = F.log_softmax(x_s, dim=1)\n        one_hot = F.one_hot(label, num_classes)\n        pow_down = one_hot * (1 - probs) + (1.0 - one_hot) * probs\n        if self.label_smoothing > 0:\n            one_hot = one_hot * (1 - self.label_smoothing) + self.label_smoothing / num_classes\n        CELoss = -1 * logprobs * one_hot\n        loss = self.alpha * torch.pow(pow_down, self.gamma) * CELoss\n        \n        loss = loss.sum(dim=1)\n        \n        if self.reduction is 'none':\n            return loss\n        \n        if self.reduction is 'mean':\n            return loss.mean()\n        \n        if self.reduction is 'sum':\n            return loss.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4.Asymmetric Loss**","metadata":{}},{"cell_type":"code","source":"class AsymmetricLossSingleLabel(nn.Module):\n    def __init__(self, gamma_pos=1, gamma_neg=4, eps: float = 0.1, reduction='none'):\n        super(AsymmetricLossSingleLabel, self).__init__()\n        self.s = 20\n        self.eps = eps\n        self.logsoftmax = nn.LogSoftmax(dim=-1)\n        self.targets_classes = []  # prevent gpu repeated memory allocation\n        self.gamma_pos = gamma_pos\n        self.gamma_neg = gamma_neg\n        self.reduction = reduction\n\n    def forward(self, inputs, target):\n        \"\"\"\"\n        Parameters\n        ----------\n        x: input logits\n        y: targets (1-hot vector)\n        \"\"\"\n\n        num_classes = inputs.size()[-1]\n        log_preds = self.logsoftmax(inputs*self.s)\n        self.targets_classes = F.one_hot(target, num_classes)\n\n        # ASL weights\n        targets = self.targets_classes\n        anti_targets = 1 - targets\n        xs_pos = torch.exp(log_preds)\n        xs_neg = 1 - xs_pos\n        xs_pos = xs_pos * targets\n        xs_neg = xs_neg * anti_targets\n        asymmetric_w = torch.pow(1 - xs_pos - xs_neg,\n                                 self.gamma_pos * targets + self.gamma_neg * anti_targets)\n        log_preds = log_preds * asymmetric_w\n\n        if self.eps > 0:  # label smoothing\n            self.targets_classes = self.targets_classes * (1 - self.eps) + self.eps / num_classes\n\n        # loss calculation\n        loss = - self.targets_classes.mul(log_preds)\n\n        loss = loss.sum(dim=-1)\n        if self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5.MetricLoss**","metadata":{}},{"cell_type":"code","source":"class MetricLoss(nn.Module):\n    def __init__(self, num_classes, margin=0.2, reduction='none'):\n        super(MetricLoss, self).__init__()\n        self.num_classes = num_classes\n        self.reduction = reduction\n        self.margin = margin\n        \n    def forward(self, embed, label):\n        embed = F.normalize(embed, dim=1)\n        onehot = F.one_hot(label, self.num_classes).float()\n        onehot = F.normalize(onehot, dim=1)\n        target = F.linear(onehot, onehot)\n        correl = F.linear(embed, embed)\n        correl = correl - target * self.margin\n        correl = torch.clamp(correl, min=0.0)\n        \n        diff = torch.pow((correl - target), 2)\n        loss = diff.sum(dim=1)\n        \n        if self.reduction is 'none':\n            return loss\n        \n        if self.reduction is 'mean':\n            return loss.mean()\n        \n        if self.reduction is 'sum':\n            return loss.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Mixed Loss**","metadata":{}},{"cell_type":"code","source":"class MixedLoss(nn.Module):\n    def __init__(self, ArcLoss, AsyLoss, MetricLoss, w1 = 1.0, w2 = 1.0, w3 = 1.0, reduction='none'):\n        super(MixedLoss, self).__init__()\n        self.ArcLoss = ArcLoss\n        self.AsyLoss = AsyLoss\n        self.MetricLoss = MetricLoss\n        self.w1 = w1\n        self.w2 = w2\n        self.w3 = w3\n        self.reduction = reduction\n        \n    def forward(self, x, embed, label):\n        loss = self.w1 * self.ArcLoss(x, label) + self.w2 * self.AsyLoss(x, label) + self.w3 * self.MetricLoss(embed, label)\n        if self.reduction is 'none':\n            return loss\n        \n        if self.reduction is 'mean':\n            return loss.mean()\n        \n        if self.reduction is 'sum':\n            return loss.sum()\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Animator","metadata":{}},{"cell_type":"markdown","source":"**Plot loss and acc**","metadata":{}},{"cell_type":"code","source":"class Animator:\n    def __init__(self, legend=None, xlim=None, ylim=[[0,None],[0,1]], fmts=('b-', 'y-', 'r-', 'g-'), figsize=(5, 5)):\n        self.w, self.h = figsize\n        self.fig, self.axes = plt.subplots(1, 2, figsize=(self.w * 2, self.h))\n        if legend is None:\n            self.legend = ['train_loss','train_acc','valid_loss','valid_acc']\n        else:\n            self.legend = legend\n        self.X = None\n        self.Y = None\n        self.xlim = xlim\n        self.ylim = ylim\n        self.fmts = fmts\n        \n        \n    def log(self):\n        \n        return self.Y\n\n        \n    def add(self, x, y):\n        if not hasattr(y, \"__len__\"):\n            y = [y]\n        n = len(y)\n        if not hasattr(x, \"__len__\"):\n            x = [x] * n\n        if self.X is None:\n            self.X = [[] for _ in range(n)]\n        if self.Y is None:\n            self.Y = [[] for _ in range(n)]\n        for i, (a, b) in enumerate(zip(x, y)):\n            self.X[i].append(a)\n            self.Y[i].append(b)\n        self.axes[0].cla()\n        self.axes[1].cla()\n        for i, (x, y, fmt) in enumerate(zip(self.X, self.Y, self.fmts)):\n            self.axes[i%2].plot(x, y, fmt, label=self.legend[i])\n            self.axes[i%2].legend(loc='best')\n            self.axes[i%2].set_xlabel('Epoch')\n            self.axes[i%2].grid(True)\n            \n        if self.xlim is not None:\n            self.axes[0].set_xlim(self.xlim)\n            self.axes[1].set_xlim(self.xlim)\n            \n        if self.ylim is not None:\n            self.axes[0].set_ylim(self.ylim[0])\n            self.axes[1].set_ylim(self.ylim[1])\n            \n        display.display(self.fig) # add display.clear_output(wait=True) to clear output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Trainer","metadata":{}},{"cell_type":"markdown","source":"**Accumulator used to accumulate loss, acc, num of elements**","metadata":{}},{"cell_type":"code","source":"class Accumulator: \n    def __init__(self, n):\n        self.data = [0.0] * n\n\n    def add(self, *args):\n        self.data = [a + float(b) for a, b in zip(self.data, args)]\n\n    def reset(self):\n        self.data = [0.0] * len(self.data)\n\n    def __getitem__(self, idx):\n        return self.data[idx]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, train_dataloader, optimizer, criterion, valid_dataloader=None, use_EMA=True):\n        self.best_acc = 0.0\n        self.model = model\n        self.use_EMA = use_EMA\n        if self.use_EMA:\n            self.model_ema = timm.utils.ModelEma(self.model, device='cuda')\n        self.model.to(device)\n        self.criterion = criterion\n        self.optimizer = optimizer\n        self.lr_scheduler = None\n        self.train_dataloader = train_dataloader\n        self.train_steps = len(train_dataloader)\n        self.valid_dataloader = valid_dataloader\n        if self.valid_dataloader is not None:\n            self.valid_steps = len(valid_dataloader)\n            \n        \n    def train(self, num_epochs, cut_mix_p=0.9, cut_mix_start=2, cut_mix_end = 28):\n        self.lr_scheduler = OneCycleLR(optimizer, max_lr=3e-4, epochs=num_epochs, steps_per_epoch=self.train_steps, \n                                       pct_start=0.3, div_factor=10, final_div_factor=10)\n                                        \n        animator = Animator(xlim=[0,num_epochs])\n        for epoch in range(num_epochs):\n            print(clr.S + f\"Epoch : {epoch}\" + clr.E)\n            train_loss, train_acc = self.train_one_epoch(epoch, cut_mix_p, cut_mix_start, cut_mix_end)\n            if valid_set is not None:\n                valid_loss, valid_acc = self.evaluate(epoch)\n                animator.add(epoch, (train_loss, train_acc, valid_loss, valid_acc))\n                if valid_acc > self.best_acc:\n                    self.best_acc = valid_acc\n                    print(clr.S + f'Found Higher Acc = {self.best_acc:.4f}. Saving State...' + clr.E)\n                    if self.use_EMA:\n                        torch.save(self.model_ema.module.state_dict(), f'HappyWhaleNet{epoch}.pth')\n                    else:   \n                        torch.save(self.model.state_dict(), f'HappyWhaleNet{epoch}.pth')\n        \n        print(clr.S + \"Train finished!\" + clr.E)\n                \n                \n    def train_one_epoch(self, epoch, cut_mix_p, cut_mix_start, cut_mix_end):\n        print(clr.S + \"--- Train ---\" + clr.E)\n        self.model.train()\n        metric = Accumulator(4)\n        train_tqdm = notebook.tqdm(self.train_dataloader, total=self.train_steps)\n        for i, data in enumerate(train_tqdm):\n            input, label = data\n            input = input.to(device)\n            label = label.to(device).reshape(-1)\n            \n            if epoch >= cut_mix_start and random.uniform(0, 1) < cut_mix_p and epoch < cut_mix_end:\n                input, label_a, label_b, lam = mixup_data(input, label, alpha=0.2)\n                output, feature = self.model(input)\n                loss = lam * self.criterion(output, feature, label_a) + (1 - lam) * self.criterion(output, feature, label_b)\n            else:\n                output, feature = self.model(input)\n                loss = self.criterion(output, feature, label)\n                \n            self.optimizer.zero_grad()\n            loss.mean().backward()\n            self.optimizer.step()\n            self.lr_scheduler.step()\n\n            if self.use_EMA:\n                self.model_ema.update(self.model)\n            \n            metric.add(float(loss.sum()), self.compute_acc(output, label), self.compute_Top5_acc(output, label), label.numel())\n            train_tqdm.set_postfix(loss = metric[0]/metric[3], acc = metric[1]/metric[3], top5acc = metric[2]/metric[3])\n\n                    \n        return metric[0] / metric[3], metric[1] / metric[3]\n         \n        \n    def evaluate(self, epoch):\n        print(clr.S + \"--- Valid ---\" + clr.E)\n        self.model.eval()\n        metric = Accumulator(4)\n        valid_tqdm = notebook.tqdm(self.valid_dataloader, total=self.valid_steps)\n        with torch.no_grad():\n            for i, data in enumerate(valid_tqdm):\n                input, label = data\n                input = input.to(device)\n                label = label.to(device)\n                label = label.reshape(-1)\n                \n                if self.use_EMA:\n                    output, feature = self.model_ema.module(input)\n                else:\n                    output, feature = self.model(input)\n                    \n                loss = self.criterion(output, feature, label)\n                metric.add(float(loss.sum()), self.compute_acc(output, label), self.compute_Top5_acc(output, label), label.numel())\n                valid_tqdm.set_postfix(loss = metric[0]/metric[3], acc = metric[1]/metric[3], top5acc = metric[2]/metric[3])\n            \n        return metric[0] / metric[3], metric[1] / metric[3]\n        \n    \n    def compute_acc(self, pred, label):\n        pred = pred.argmax(axis=1)\n        cmp = pred.type(label.dtype) == label\n        \n        return float(cmp.type(label.dtype).sum())\n    \n    \n    def compute_Top5_acc(self, pred, label):\n        top5 = torch.topk(pred, dim=1, k=5)\n        cmp = top5.indices == label.reshape(-1,1)\n        \n        return float(cmp.type(label.dtype).sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🐬 Prepare training","metadata":{}},{"cell_type":"code","source":"def DataProvider(train_set, valid_set=None, batch_size=32, num_workers=2):\n    train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n    if valid_set is not None:\n        valid_dataloader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n        return {'train':train_dataloader, 'valid':valid_dataloader}\n    else:\n        return {'train':train_dataloader}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def str2int(s):\n    if type(s) != str:\n        return 'nan'\n    xmin, ymin, xmax, ymax = s.strip().split(' ')\n    \n    return [int(xmin.strip()), int(ymin.strip()), int(xmax.strip()), int(ymax.strip())]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/kfolds-cropped-whale-train-csv/kfolds_cropped_whale_train.csv')\ndf['box'] = df['box'].apply(str2int)\n\nNUM_CLASSES = len(df.groupby('individual_id'))\n\nprint(f'Dataset size : {len(df)}')\nprint(f'Number of individuals : {NUM_CLASSES}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df[df['kfold:0'] == 'train'].reset_index(drop=True)\nvalid_df = df[df['kfold:0'] == 'valid'].reset_index(drop=True)\n\ntrain_df = train_df[:5000]\n\ntrain_set = HappyWhaleDataset(train_df, TRAIN_IMG_DIR, transform=transforms['train'], crop_p=1.0)\nvalid_set = HappyWhaleDataset(valid_df, TRAIN_IMG_DIR, transform=transforms['valid'], crop_p=1.0)\n\nprint(f'train_set size : {len(train_set)}')\nprint(f'valid_set size : {len(valid_set)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arcloss = ArcLoss()\nasyloss = AsymmetricLossSingleLabel()\nmetricloss = MetricLoss(num_classes=NUM_CLASSES)\n\nmixedloss = MixedLoss(ArcLoss=arcloss, AsyLoss=asyloss, MetricLoss=metricloss, w1 = 1, w2 = 1, w3 = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Pretrained_U2Net = r'../input/pretrained-u2net/u2net.pth'\n# model = SegEffWhaleNetV2(backbone=cfg['Effnet_B0'], num_classes=NUM_CLASSES, embedding_size=512, unet_path=Pretrained_U2Net, backbone_pretrained=True)\nmodel = EffWhaleNet(backbone=cfg['Effnet_B0'], num_classes=NUM_CLASSES, embedding_size=512, backbone_pretrained=True)\n\nn_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f'number of params : {n_parameters//1e6}M')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"provider = DataProvider(train_set, valid_set, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS)\n\noptimizer = timm.optim.AdamP((param for param in model.parameters() if param.requires_grad), lr=3e-4, weight_decay=2e-5)\n\nwhale_trainer =Trainer(model, provider['train'], optimizer, criterion=mixedloss, valid_dataloader=provider['valid'], use_EMA=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"whale_trainer.train(num_epochs = 10, cut_mix_p = 0, cut_mix_start = 0, cut_mix_end = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in model.backbone.parameters():\n    param.requires_grad = False\n    \nn_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f'number of params : {n_parameters//1e6}M')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_group_1 = [param for param in model.backbone.parameters() if param.requires_grad]\nparam_group_2 = []\n\nfor param in model.GeM.parameters():\n    param_group_2.append(param)\n    \nfor param in model.Embedding.parameters():\n    param_group_2.append(param)\n    \nfor param in model.CosProduct.parameters():\n    param_group_2.append(param)\n    \noptimizer = timm.optim.AdamW([{'params':param_group_1, 'lr':LR/10 }, \n                              {'params':param_group_2}], lr = LR, weight_decay = 2e-5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1.EffV1 weight_decay=1e-6 epoch=4 acc=0.066 overfitting**","metadata":{}},{"cell_type":"markdown","source":"**2.EffV1 weight_decay=1e-5 epoch=15**","metadata":{}},{"cell_type":"markdown","source":"# 🐬 Analysis","metadata":{}},{"cell_type":"markdown","source":"# 🐬 Test","metadata":{}},{"cell_type":"code","source":"individual_label_map = df[['individual_id', 'label']].drop_duplicates()\nindividual_label_map = individual_label_map.append({'individual_id':'new_individual', 'label':NUM_CLASSES}, ignore_index=True)\nindividual_label_map = individual_label_map.sort_values(by='label', ignore_index=True)\nindividual_label_map.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WhaleTestDataset(Dataset):\n    def __init__(self, path:str, df:pd.DataFrame, crop=False, transform=None):\n        self.path = path\n        self.df = df\n        self.crop = crop\n        self.transform = transform\n        \n    def __getitem__(self, idx):\n        img_path = os.path.join(self.path, self.df['image'][idx])\n        img = Image.open(img_path).convert('RGB')\n        img = np.array(img)\n        \n        if self.crop:\n            if self.df['box'][idx] == 'nan':\n                pass\n            else:\n                xmin, ymin, xmax, ymax = self.df['box'][idx]\n                image = image[ymin:ymax, xmin:xmax, :]\n        \n        if self.transform is not None:\n            img = self.transform(image=img)['image']\n\n        img = img.transpose(2, 0, 1)\n        img = torch.tensor(img)\n        \n        return img ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Test:\n    def __init__(self, sample_df_path:str, image_path:str, model, num_classes, transform=None, TTA_num=4, threshold=0.3):\n        self.sample_df = pd.read_csv(sample_df_path)\n        self.dataset = WhaleTestDatasetV2(path=image_path, df=self.sample_df, transform=transform)\n        self.size = len(self.sample_df)\n        self.model = model\n        self.model.to(device)\n        self.model.eval()\n        self.num_classes = num_classes\n        self.threshold = threshold\n        self.TTA_num = TTA_num\n        \n        \n    def test(self, save_path='sample_submission.csv'):\n        for i in range(self.size):\n            pred = self.TTA(idx=i)\n            top5_label, top5_value = self.get_Top5(pred)\n            individuals = individual_label_map['individual_id'][np.array(top5_label.cpu())].tolist()\n            individuals = ' '.join(individuals)\n            self.sample_df['predictions'][i] = individuals\n            print(f'\\r{i}/{self.size}|{top5_value.detach().cpu()}|{individuals}', end=' ')\n        \n        self.sample_df.to_csv(save_path, index=0)\n            \n            \n    def TTA(self, idx):\n        pred = torch.zeros(self.num_classes).to(device)\n        for time in range(self.TTA_num):\n            img = self.dataset[idx]\n            img = TTA_transform[str(time)](image=img)['image']\n            img = torch.tensor(img).unsqueeze(0)\n            img = img.to(device)\n            output, _ = self.model(img)\n            pred += output.squeeze(0)\n        pred /= self.TTA_num\n        \n        return pred     \n           \n        \n    def get_Top5(self, pred):\n        pred = torch.cat([pred, torch.tensor([self.threshold]).to(device)])\n        top5 = torch.topk(pred, k=5)\n        top5_value = top5.values\n        top5_label = top5.indices\n            \n        return top5_label, top5_value","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**1. Model_1 score : 0.62**","metadata":{}},{"cell_type":"code","source":"model_path = '../input/pretrained-happywhale-segconvs/SegConvS_HappyWhaleNet18_384.pth'\nmodel = SegConvWhaleNet(backbone='convnext_small', num_classes=NUM_CLASSES, embedding_size=512, backbone_pretrained=False)\nmodel.load_state_dict(torch.load(model_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = '../input/whale2-cropped-dataset/cropped_test_images/cropped_test_images'\nsample_df_path = '../input/happy-whale-and-dolphin/sample_submission.csv'\n\ntest_tool = Test(sample_df_path, image_path, model, num_classes = NUM_CLASSES, transform=transforms['TTA'], TTA_num=5, threshold=0.35)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tool.test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}