{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.00721,"end_time":"2022-06-29T14:47:08.527039","exception":false,"start_time":"2022-06-29T14:47:08.519829","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%capture\n!pip install timm\n!pip install randaugment\n!pip install torchtoolbox==0.0.2","metadata":{"papermill":{"duration":32.643568,"end_time":"2022-06-29T14:47:41.176974","exception":false,"start_time":"2022-06-29T14:47:08.533406","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:15.855090Z","iopub.execute_input":"2022-06-30T15:12:15.855486Z","iopub.status.idle":"2022-06-30T15:12:48.340290Z","shell.execute_reply.started":"2022-06-30T15:12:15.855408Z","shell.execute_reply":"2022-06-30T15:12:48.339036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\nimport math\nimport copy\nimport itertools\nimport numpy as np\nfrom PIL import Image\nfrom copy import deepcopy\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom collections import OrderedDict\nfrom tqdm import tqdm\nimport random\nimport pandas as pd\nimport cv2\n\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport torch.nn.functional as F\n\nimport timm\nfrom timm.models.layers.activations import *\nfrom timm.loss import LabelSmoothingCrossEntropy\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report, roc_auc_score, roc_curve\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom randaugment import RandAugment, ImageNetPolicy, Cutout\n\nfrom torchtoolbox.tools import mixup_data, mixup_criterion\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"papermill":{"duration":10.716593,"end_time":"2022-06-29T14:47:51.900063","exception":false,"start_time":"2022-06-29T14:47:41.183470","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:48.343129Z","iopub.execute_input":"2022-06-30T15:12:48.343417Z","iopub.status.idle":"2022-06-30T15:12:56.880898Z","shell.execute_reply.started":"2022-06-30T15:12:48.343389Z","shell.execute_reply":"2022-06-30T15:12:56.879896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{"papermill":{"duration":0.006365,"end_time":"2022-06-29T14:47:51.913091","exception":false,"start_time":"2022-06-29T14:47:51.906726","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    seed = 108\n    backbone = 'resnet50_ibn_a'\n#     num_classes = 100\n    batch_size = 8\n    img_size = 1024\n    num_epochs = 12\n    num_folds = 5\n    folds = [2]\n    tta = 5\n    \n    optim = 'adamW'\n    lr = 1e-4\n    weight_decay = 1e-3\n    eta_min = 1e-5\n    \n    \"\"\"ArcFace parameter\"\"\"\n    num_classes = 100\n    embedding_size = 1024\n    S, M = 30.0, 0.3 # S:cosine scale in arcloss. M:arg penalty\n    EASY_MERGING, LS_EPS = False, 0.0\n    \n    \"\"\"mixup parameter\"\"\"\n    mix_up = True\n    mixup_prob = 0.25\n    alpha = 1\n    \n    \"\"\"cutmix parameter\"\"\"\n    cut_mix = True\n    cutmix_prob = 0.25\n    beta = 1\n    \n    root_in = '/kaggle/input/small-jpegs-fgvc'\n    root_out = '/kaggle/working/'\n    df_file = 'train_df.csv'\n    submission = 'sample_submission.csv'","metadata":{"papermill":{"duration":0.01787,"end_time":"2022-06-29T14:47:51.937105","exception":false,"start_time":"2022-06-29T14:47:51.919235","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:56.882623Z","iopub.execute_input":"2022-06-30T15:12:56.883319Z","iopub.status.idle":"2022-06-30T15:12:56.892354Z","shell.execute_reply.started":"2022-06-30T15:12:56.883282Z","shell.execute_reply":"2022-06-30T15:12:56.891391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(CFG.seed)","metadata":{"papermill":{"duration":0.018338,"end_time":"2022-06-29T14:47:51.961585","exception":false,"start_time":"2022-06-29T14:47:51.943247","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:56.895469Z","iopub.execute_input":"2022-06-30T15:12:56.896267Z","iopub.status.idle":"2022-06-30T15:12:56.907315Z","shell.execute_reply.started":"2022-06-30T15:12:56.896166Z","shell.execute_reply":"2022-06-30T15:12:56.906356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Accumulator():\n    '''A counter util, which count the float value of the input'''\n    def __init__(self, nums):\n        self.metric = list(torch.zeros((nums,)).numpy())\n    def __getitem__(self, index):\n        return self.metric[index]\n    def add(self, *args):\n        for i, item in enumerate(args):\n            self.metric[i] += float(item)\n\n\ndef accuracy(y_hat, y):\n    '''used to count the right type'''\n    y_hat = y_hat.exp().argmax(dim=1)\n    y_hat.reshape((-1))\n    y.reshape((-1))\n    return accuracy_score(y.cpu().numpy(), y_hat.cpu().numpy(), normalize=False)\n\n\ndef evaluate_accuracy(net, data_iter):\n    '''Evalue the valid dataset'''\n    net.eval()\n    softmax = nn.Softmax(dim=1)\n    metric = Accumulator(2)\n    with torch.no_grad():\n        for X, y in data_iter:\n            if isinstance(X, list):\n                X = [x.to(device) for x in X]\n            else:\n                X = X.to(device)\n            y.to(device)\n            #             y_hat = net(X, y)\n            with torch.cuda.amp.autocast(enabled=True):\n                embeddings = net.extract(X)\n                y_hat = softmax(CFG.S * F.linear(F.normalize(embeddings), F.normalize(net.fc.weight)))\n                # y_hat = net(X, y)\n            metric.add(accuracy(y_hat, y), y.numel())\n    return metric[0] / metric[1]","metadata":{"papermill":{"duration":0.021698,"end_time":"2022-06-29T14:47:51.989364","exception":false,"start_time":"2022-06-29T14:47:51.967666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:56.910805Z","iopub.execute_input":"2022-06-30T15:12:56.911073Z","iopub.status.idle":"2022-06-30T15:12:56.923915Z","shell.execute_reply.started":"2022-06-30T15:12:56.911050Z","shell.execute_reply":"2022-06-30T15:12:56.922987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2  ","metadata":{"papermill":{"duration":0.016436,"end_time":"2022-06-29T14:47:52.011665","exception":false,"start_time":"2022-06-29T14:47:51.995229","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:56.925265Z","iopub.execute_input":"2022-06-30T15:12:56.925856Z","iopub.status.idle":"2022-06-30T15:12:56.935964Z","shell.execute_reply.started":"2022-06-30T15:12:56.925820Z","shell.execute_reply":"2022-06-30T15:12:56.934489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{"papermill":{"duration":0.00614,"end_time":"2022-06-29T14:47:52.023721","exception":false,"start_time":"2022-06-29T14:47:52.017581","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preparation(input_csv, output_csv):\n    \"\"\"transfer cultivar into index\"\"\"\n    df = pd.read_csv(input_csv, index_col='image')\n    df['label_index'] = torch.zeros((df.shape[0])).type(torch.int32).numpy()\n    labels_map = dict()\n    for i, label in enumerate(df['cultivar'].unique()):\n        labels_map[i] = label\n        df.loc[df.cultivar == label, 'label_index'] = i\n    df.to_csv(output_csv)\n    return labels_map\n\nlabels_map = preparation(os.path.join(CFG.root_in, 'train_cultivar_mapping.csv'), os.path.join(CFG.root_out, CFG.df_file))\ntrain_df = pd.read_csv(os.path.join(CFG.root_out, CFG.df_file))","metadata":{"papermill":{"duration":0.601885,"end_time":"2022-06-29T14:47:52.631936","exception":false,"start_time":"2022-06-29T14:47:52.030051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:56.937544Z","iopub.execute_input":"2022-06-30T15:12:56.938422Z","iopub.status.idle":"2022-06-30T15:12:57.686639Z","shell.execute_reply.started":"2022-06-30T15:12:56.938384Z","shell.execute_reply":"2022-06-30T15:12:57.685278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.006149,"end_time":"2022-06-29T14:47:52.644843","exception":false,"start_time":"2022-06-29T14:47:52.638694","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_transforms = {\n    'train': A.Compose([\n        A.CLAHE(clip_limit=40, tile_grid_size=(10, 10),p=1.0),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=20, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101, p=0.5),\n        # A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n        A.OneOf([A.RandomBrightness(limit=0.1, p=1), A.RandomContrast(limit=0.1, p=1)]),\n        A.Normalize(),\n        ToTensorV2(p=1.0),\n    ]),\n    'valid': A.Compose([\n        A.CLAHE(clip_limit=40, tile_grid_size=(10, 10),p=1.0),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(),\n        ToTensorV2(p=1.0),\n    ]),\n    'test': A.Compose([\n        A.CLAHE(clip_limit=40, tile_grid_size=(10, 10),p=1.0),\n        A.Resize(CFG.img_size, CFG.img_size),\n        A.Normalize(),\n        ToTensorV2(p=1.0),\n    ])\n}","metadata":{"papermill":{"duration":0.022813,"end_time":"2022-06-29T14:47:52.673469","exception":false,"start_time":"2022-06-29T14:47:52.650656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.688187Z","iopub.execute_input":"2022-06-30T15:12:57.688825Z","iopub.status.idle":"2022-06-30T15:12:57.703849Z","shell.execute_reply.started":"2022-06-30T15:12:57.688783Z","shell.execute_reply":"2022-06-30T15:12:57.702758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumDataset(Dataset):\n    def __init__(self, df, root_dir=CFG.root_in, transform=None, folder='train'):\n        super().__init__()\n        self.df = df\n        self.images_path = df['image'].values\n        self.labels = df['label_index'].values\n        self.transform = transform\n        self.folder = 'train'\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        image_path, label = self.images_path[index], self.labels[index]\n        image_path = os.path.join(CFG.root_in, self.folder, image_path)\n        img = Image.open(image_path)\n        img = np.array(img)\n        if self.transform:\n            aug = self.transform(image=img)\n            img = aug['image']\n        return img, label\n    \nclass SorghumTestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        super().__init__()\n        self.df = df\n        self.transform = transform\n        self.images_path = df['filename'].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        filename = self.images_path[index].split('.')[0]\n        img = Image.open(os.path.join(CFG.root_in, 'test', filename+'.jpeg'))\n        img = np.array(img)\n        if self.transform:\n            aug = self.transform(image=img)\n            img = aug['image']\n        return img","metadata":{"papermill":{"duration":0.021088,"end_time":"2022-06-29T14:47:52.700513","exception":false,"start_time":"2022-06-29T14:47:52.679425","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.705305Z","iopub.execute_input":"2022-06-30T15:12:57.705956Z","iopub.status.idle":"2022-06-30T15:12:57.725840Z","shell.execute_reply.started":"2022-06-30T15:12:57.705920Z","shell.execute_reply":"2022-06-30T15:12:57.724782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split KFolds","metadata":{"papermill":{"duration":0.005858,"end_time":"2022-06-29T14:47:52.712415","exception":false,"start_time":"2022-06-29T14:47:52.706557","status":"completed"},"tags":[]}},{"cell_type":"code","source":"folds = train_df.copy()\nkfolds = StratifiedKFold(n_splits=CFG.num_folds, random_state=CFG.seed, shuffle=True)\n\nfor n, (train_idx, val_idx) in enumerate(kfolds.split(folds, folds['label_index'])):\n    folds.loc[val_idx, 'fold'] = int(n)\n    \nfolds['fold'] = folds['fold'].astype(int)","metadata":{"papermill":{"duration":0.033072,"end_time":"2022-06-29T14:47:52.751453","exception":false,"start_time":"2022-06-29T14:47:52.718381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.736018Z","iopub.execute_input":"2022-06-30T15:12:57.736867Z","iopub.status.idle":"2022-06-30T15:12:57.760113Z","shell.execute_reply.started":"2022-06-30T15:12:57.736831Z","shell.execute_reply":"2022-06-30T15:12:57.759100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.005705,"end_time":"2022-06-29T14:47:52.763689","exception":false,"start_time":"2022-06-29T14:47:52.757984","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class se_block(nn.Module):\n    def __init__(self, channel, ratio=16):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n                nn.Linear(channel, channel//ratio, bias=False),\n                nn.ReLU(inplace=True),\n                nn.Linear(channel//ratio, channel, bias=False),\n                nn.Sigmoid()\n        )\n    \n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y","metadata":{"papermill":{"duration":0.017078,"end_time":"2022-06-29T14:47:52.786801","exception":false,"start_time":"2022-06-29T14:47:52.769723","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.761572Z","iopub.execute_input":"2022-06-30T15:12:57.761929Z","iopub.status.idle":"2022-06-30T15:12:57.769081Z","shell.execute_reply.started":"2022-06-30T15:12:57.761895Z","shell.execute_reply":"2022-06-30T15:12:57.767786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n    Args:\n        in_features: size of each input sample\n        out_features: size of each output sample\n        s: norm of input feature\n        m: margin\n        cos(theta + m)\n    \"\"\"\n\n    def __init__(\n            self,\n            in_features: int,\n            out_features: int,\n            s: float,\n            m: float,\n            easy_margin: bool,\n            ls_eps: float,\n    ):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input: torch.Tensor, label: torch.Tensor) -> torch.Tensor:\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        # Enable 16 bit precision\n        cosine = cosine.to(torch.float32)\n\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size()).to(device)\n        one_hot.scatter_(1, label.view(-1, 1).long().to(device), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output   ","metadata":{"papermill":{"duration":0.022099,"end_time":"2022-06-29T14:47:52.814694","exception":false,"start_time":"2022-06-29T14:47:52.792595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.770603Z","iopub.execute_input":"2022-06-30T15:12:57.771286Z","iopub.status.idle":"2022-06-30T15:12:57.787422Z","shell.execute_reply.started":"2022-06-30T15:12:57.771230Z","shell.execute_reply":"2022-06-30T15:12:57.786466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_urls = {\n    'resnet18_ibn_a': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet18_ibn_a-2f571257.pth',\n    'resnet34_ibn_a': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet34_ibn_a-94bc1577.pth',\n    'resnet50_ibn_a': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet50_ibn_a-d9d0bb7b.pth',\n    'resnet101_ibn_a': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet101_ibn_a-59ea0ac6.pth',\n    'resnet18_ibn_b': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet18_ibn_b-bc2f3c11.pth',\n    'resnet34_ibn_b': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet34_ibn_b-04134c37.pth',\n    'resnet50_ibn_b': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet50_ibn_b-9ca61e85.pth',\n    'resnet101_ibn_b': 'https://github.com/XingangPan/IBN-Net/releases/download/v1.0/resnet101_ibn_b-c55f6dba.pth',\n}\n\nclass IBN(nn.Module):\n    r\"\"\"Instance-Batch Normalization layer from\n    `\"Two at Once: Enhancing Learning and Generalization Capacities via IBN-Net\"\n    <https://arxiv.org/pdf/1807.09441.pdf>`\n    Args:\n        planes (int): Number of channels for the input tensor\n        ratio (float): Ratio of instance normalization in the IBN layer\n    \"\"\"\n    def __init__(self, planes, ratio=0.5):\n        super(IBN, self).__init__()\n        self.half = int(planes * ratio)\n        self.IN = nn.InstanceNorm2d(self.half, affine=True)\n        self.BN = nn.BatchNorm2d(planes - self.half)\n\n    def forward(self, x):\n        split = torch.split(x, self.half, 1)\n        out1 = self.IN(split[0].contiguous())\n        out2 = self.BN(split[1].contiguous())\n        out = torch.cat((out1, out2), 1)\n        return out\n\nclass BasicBlock_IBN(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, ibn=None, stride=1, downsample=None):\n        super(BasicBlock_IBN, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        if ibn == 'a':\n            self.bn1 = IBN(planes)\n        else:\n            self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.IN = nn.InstanceNorm2d(planes, affine=True) if ibn == 'b' else None\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        if self.IN is not None:\n            out = self.IN(out)\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck_IBN(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, ibn=None, stride=1, downsample=None):\n        super(Bottleneck_IBN, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        if ibn == 'a':\n            self.bn1 = IBN(planes)\n        else:\n            self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n        self.IN = nn.InstanceNorm2d(planes * 4, affine=True) if ibn == 'b' else None\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        if self.IN is not None:\n            out = self.IN(out)\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet_IBN(nn.Module):\n\n    def __init__(self,\n                 block,\n                 layers,\n                 ibn_cfg=('a', 'a', 'a', None),\n                 num_classes=1000):\n        self.inplanes = 64\n        super(ResNet_IBN, self).__init__()\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        if ibn_cfg[0] == 'b':\n            self.bn1 = nn.InstanceNorm2d(64, affine=True)\n        else:\n            self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], ibn=ibn_cfg[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, ibn=ibn_cfg[1])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, ibn=ibn_cfg[2])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=1, ibn=ibn_cfg[3])\n        self.avgpool = nn.AvgPool2d(7)\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n                m.weight.data.normal_(0, math.sqrt(2. / n))\n            elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.InstanceNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def _make_layer(self, block, planes, blocks, stride=1, ibn=None):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes,\n                            None if ibn == 'b' else ibn,\n                            stride, downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes,\n                                None if (ibn == 'b' and i < blocks-1) else ibn))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        # x = self.avgpool(x)\n        # x = x.view(x.size(0), -1)\n        # x = self.fc(x)\n\n        return x\n\n\ndef resnet50_ibn_a(pretrained=False, **kwargs):\n    \"\"\"Constructs a ResNet-50-IBN-a model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet_IBN(block=Bottleneck_IBN,\n                       layers=[3, 4, 6, 3],\n                       ibn_cfg=('a', 'a', 'a', None),\n                       **kwargs)\n    if pretrained:\n        model.load_state_dict(torch.hub.load_state_dict_from_url(model_urls['resnet50_ibn_a']))\n    return model","metadata":{"papermill":{"duration":0.047581,"end_time":"2022-06-29T14:47:52.868177","exception":false,"start_time":"2022-06-29T14:47:52.820596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.789408Z","iopub.execute_input":"2022-06-30T15:12:57.789779Z","iopub.status.idle":"2022-06-30T15:12:57.832793Z","shell.execute_reply.started":"2022-06-30T15:12:57.789735Z","shell.execute_reply":"2022-06-30T15:12:57.831850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumModel(nn.Module):\n    def __init__(self, model_name, embedding_size, num_classes):\n        super(SorghumModel, self).__init__()\n        self.model = resnet50_ibn_a(pretrained=True)\n        in_features = 2048\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        \n        self.enhance = se_block(channel=in_features, ratio=8)\n        self.multiple_dropout = [nn.Dropout(0.25) for i in range(5)]\n        self.embedding = nn.Linear(in_features, embedding_size)\n        \n        # bnnneck\n        self.bottleneck = nn.BatchNorm1d(embedding_size)\n        self.bottleneck.bias.requires_grad_(False)\n        self.pr = nn.PReLU()\n        \n        self.fc = ArcMarginProduct(embedding_size, num_classes, CFG.S, CFG.M, CFG.EASY_MERGING, CFG.LS_EPS)\n        \n    def forward(self, images, labels):\n        features = self.model(images)\n        features = self.enhance(features)\n        pooled_features = self.pooling(features).flatten(1)\n        pooled_features_dropout = torch.zeros((pooled_features.shape)).to(device)\n        for i in range(5):\n            pooled_features_dropout += self.multiple_dropout[i](pooled_features)\n        pooled_features_dropout /= 5\n        embedding = self.embedding(pooled_features_dropout)\n        embedding = self.bottleneck(embedding)\n        embedding = self.pr(embedding)\n        output = self.fc(embedding, labels)\n        return output\n    \n    def extract(self, images):\n        features = self.model(images)\n        features = self.enhance(features)\n        pooled_features = self.pooling(features).flatten(1)\n        embedding = self.embedding(pooled_features)\n        embedding = self.bottleneck(embedding)\n        embedding = self.pr(embedding)\n        return embedding","metadata":{"papermill":{"duration":0.021105,"end_time":"2022-06-29T14:47:52.895504","exception":false,"start_time":"2022-06-29T14:47:52.874399","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.835847Z","iopub.execute_input":"2022-06-30T15:12:57.836096Z","iopub.status.idle":"2022-06-30T15:12:57.850473Z","shell.execute_reply.started":"2022-06-30T15:12:57.836072Z","shell.execute_reply":"2022-06-30T15:12:57.849556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{"papermill":{"duration":0.005783,"end_time":"2022-06-29T14:47:52.907195","exception":false,"start_time":"2022-06-29T14:47:52.901412","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_optim(net):\n    if CFG.optim == 'sgd':\n        optimizer = optim.SGD((param for param in net.parameters() if param.requires_grad), lr=CFG.lr,\n                                    weight_decay=CFG.weight_decay)\n    elif CFG.optim == 'adam':\n        optimizer = optim.Adam((param for param in net.parameters() if param.requires_grad), lr=CFG.lr,\n                                     weight_decay=CFG.weight_decay)\n    elif CFG.optim == 'adamW':\n        optimizer = optim.AdamW((param for param in net.parameters() if param.requires_grad), lr=CFG.lr,\n                                      weight_decay=CFG.weight_decay)\n        \n    return optimizer","metadata":{"papermill":{"duration":0.015659,"end_time":"2022-06-29T14:47:52.928870","exception":false,"start_time":"2022-06-29T14:47:52.913211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.852892Z","iopub.execute_input":"2022-06-30T15:12:57.853669Z","iopub.status.idle":"2022-06-30T15:12:57.873515Z","shell.execute_reply.started":"2022-06-30T15:12:57.853634Z","shell.execute_reply":"2022-06-30T15:12:57.872616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(net, loss, train_loader, val_loader, fold):\n    num_batches = len(train_loader)\n    best_accuracy = 0\n    \n    if CFG.optim == 'sgd':\n        optimizer = torch.optim.SGD((param for param in net.parameters() if param.requires_grad), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    elif CFG.optim == 'adam':\n        optimizer = torch.optim.Adam((param for param in net.parameters() if param.requires_grad), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    elif CFG.optim == 'adamW':\n        optimizer = torch.optim.AdamW((param for param in net.parameters() if param.requires_grad), lr=CFG.lr, weight_decay=CFG.weight_decay)\n\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=3, T_mult=2, eta_min=CFG.eta_min, last_epoch=-1)\n    \n    for epoch in range(CFG.num_epochs):\n        print('-'*5, f'Epoch {epoch+1}/{CFG.num_epochs}', '-'*5)\n        net.train()\n        metric = Accumulator(3)\n        for i, (images, targets) in enumerate(tqdm(train_loader)):\n            images = images.to(device)\n            targets = targets.to(device)\n                    \n            with torch.cuda.amp.autocast(enabled=True):\n                # mixup and cutmix       \n                rand = np.random.rand()\n                if CFG.mix_up and (rand < CFG.mixup_prob):    \n                    images, labels_a, labels_b, lam = mixup_data(images, targets, CFG.alpha)\n                    y_hat = net(images, targets)\n                    l = mixup_criterion(loss, y_hat, labels_a, labels_b, lam)\n                elif CFG.cut_mix and (CFG.mixup_prob< rand < (CFG.mixup_prob + CFG.cutmix_prob)):\n                    lam = np.random.beta(CFG.beta, CFG.beta)\n                    rand_index = torch.randperm(images.size()[0])\n                    target_a = targets\n                    target_b = targets[rand_index]\n                    bbx1, bby1, bbx2, bby2 = rand_bbox(images.size(), lam)\n                    images[:, :, bbx1:bbx2, bby1:bby2] = images[rand_index, :, bbx1:bbx2, bby1:bby2]\n                    # adjust lambda to exactly match pixel ratio\n                    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (images.size()[-1] * images.size()[-2]))\n                    # compute output\n                    y_hat = net(images, targets)\n                    l = loss(y_hat, target_a) * lam + loss(y_hat, target_b) * (1. - lam)\n                else:\n                    y_hat = net(images, targets)\n                    l = loss(y_hat, targets)\n\n            optimizer.zero_grad()\n            l.backward()\n            nn.utils.clip_grad_norm_(net.parameters(), max_norm=20, norm_type=2)\n            optimizer.step()\n\n            \n            with torch.no_grad():\n                metric.add(l * images.shape[0], accuracy(y_hat, targets), images.shape[0])\n            train_l = metric[0] / metric[2]\n            train_acc = metric[1] / metric[2]\n            if (i + 1) % (num_batches // 1) == 0 or i == num_batches - 1:\n                print('Train: loss: {:.4f} accuracy: {:.4f}'.format(train_l, train_acc)) \n                Value_train_l.append(train_l)\n                Value_train_acc.append(train_acc)\n                Value_test_acc.append(None)\n                Time.append(epoch + (i + 1) / num_batches)\n\n        scheduler.step()\n\n        test_acc = evaluate_accuracy(net, val_loader)\n\n        print('Valid: accuracy: {:.4f} lr: {:.4f}'.format(test_acc, optimizer.param_groups[0]['lr']))\n        Value_train_l.append(None)\n        Value_train_acc.append(None)\n        Value_test_acc.append(test_acc)\n        Time.append(epoch + 1)\n        \n        if test_acc >= best_accuracy:\n            best_accuracy = test_acc\n            torch.save(net.state_dict(), f'fold{fold}_best.pth')\n        \n    torch.save(net.state_dict(), f'fold{fold}_last.pth')\n    print(f'loss {train_l:.3f}, train acc {train_acc:.3f}, ' f'test acc {test_acc:.3f}')\n    torch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.032855,"end_time":"2022-06-29T14:47:52.967623","exception":false,"start_time":"2022-06-29T14:47:52.934768","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.875087Z","iopub.execute_input":"2022-06-30T15:12:57.875507Z","iopub.status.idle":"2022-06-30T15:12:57.902617Z","shell.execute_reply.started":"2022-06-30T15:12:57.875471Z","shell.execute_reply":"2022-06-30T15:12:57.901664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in CFG.folds:\n    print('='*5, f'Fold {fold}', '='*5); print()\n    Value_train_l = list()\n    Value_train_acc = list()\n    Value_test_acc = list()\n    Time = list()\n\n    train_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[train_idx].reset_index(drop=True)\n    val_folds = folds.loc[val_idx].reset_index(drop=True)\n\n    train_dataset = SorghumDataset(train_folds, transform=data_transforms['train'])\n    val_dataset = SorghumDataset(val_folds, transform=data_transforms['valid'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=2, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n\n    path = f'model_fold{fold}.pth'\n    net = SorghumModel(CFG.backbone, CFG.embedding_size, CFG.num_classes).to(device)\n    loss = nn.CrossEntropyLoss()\n    train_model(net, loss, train_loader, val_loader, fold)","metadata":{"papermill":{"duration":35178.680504,"end_time":"2022-06-30T00:34:11.654281","exception":false,"start_time":"2022-06-29T14:47:52.973777","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-30T15:12:57.905551Z","iopub.execute_input":"2022-06-30T15:12:57.906118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_l = np.array(Value_train_l)\ntrain_acc = np.array(Value_train_acc)\ntest_acc = np.array(Value_test_acc)\nt = np.array(Time)\n\nnp.save('train_loss.npy', train_l)\nnp.save('train_acc.npy', train_acc)\nnp.save('test_acc.npy', test_acc)\nnp.save('time.npy', t)","metadata":{"papermill":{"duration":1.65542,"end_time":"2022-06-30T00:34:14.722859","exception":false,"start_time":"2022-06-30T00:34:13.067439","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}