{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌱🌿🌾Sorghum PyTorch TTA Inference Tutorial🚀\n\nThis notbook is ***the inference of*** the [Sorghum Speed UP](https://www.kaggle.com/code/leoooo333/sorghum-speed-up) tutorial.  \n\nThanks to the small jpegs Sorghum images from https://www.kaggle.com/datasets/mithilsalunkhe/small-jpegs-fgvc\n\nIf you have any ***question*** about my baseline, please *feel free* to ***make a comment***. I will reply as soon as possible! If you like it, please **upvote**👏👏👏\n\n\n## Inference result:\n>##### The first fold in 5-Fold model, without TTA, get LB **Acc@1: 82.9%**\n>##### The 1,2 fold mix up(Avg), without TTA, get LB **Acc@1: 85.5%**\n>##### All train image(instead of k-fold) model, without TTA, get LB **Acc@1: 85.6%**\n>##### All train image(instead of k-fold) model, use flip and crop TTA, get LB **Acc@1: 86.3%**\n>##### The 1,2,3 fold in 5-Fold and all-enrolled model mixup, use flip and crop TTA, get LB **Acc@1: 87.7%**\n\n## main idea\n+ *pre-process* the images(use CLAHE)\n+ visualize and check the pre-process\n+ use the pretrained model(import timm)\n+ images augmentation\n+ [train the model with DDP (and mix precise)](https://www.kaggle.com/code/leoooo333/sorghum-speed-up)\n+ Inference\n\n## tricks\n\n#### To learn about training tricks\nClick here [Sorghum Speed UP](https://www.kaggle.com/code/leoooo333/sorghum-speed-up) !\n\n#### Inference\n+ TTA : test time augmentation, make different trivial augmentations on test images, mix up all results, and get the average.","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:20:08.083665Z","iopub.execute_input":"2022-04-21T03:20:08.083988Z","iopub.status.idle":"2022-04-21T03:20:08.104463Z","shell.execute_reply.started":"2022-04-21T03:20:08.083953Z","shell.execute_reply":"2022-04-21T03:20:08.103329Z"}}},{"cell_type":"code","source":"!pip install seaborn\n!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:20:50.039593Z","iopub.execute_input":"2022-04-21T02:20:50.039925Z","iopub.status.idle":"2022-04-21T02:21:12.258504Z","shell.execute_reply.started":"2022-04-21T02:20:50.03989Z","shell.execute_reply":"2022-04-21T02:21:12.257269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import transforms, models\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom PIL import Image as Img\nfrom tqdm import tqdm\nimport os\nfrom sklearn.metrics import accuracy_score\nimport timm\nfrom tqdm import tqdm  \n\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, LambdaLR\n\nimport torchvision\nimport matplotlib.pyplot as plt\nimport re\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nimport pytorch_lightning as pl\nimport seaborn as sns\nimport cv2 as cv\nimport numpy as np\nimport torch.nn.functional as F\n\nimport torch.distributed as dist\nimport torch.multiprocessing as mp\nfrom torch.nn.parallel import DistributedDataParallel as DDP\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.tensorboard import SummaryWriter\n\nimport numpy as np\nimport math","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:14.657733Z","iopub.execute_input":"2022-04-21T02:22:14.658227Z","iopub.status.idle":"2022-04-21T02:22:16.505785Z","shell.execute_reply.started":"2022-04-21T02:22:14.658189Z","shell.execute_reply":"2022-04-21T02:22:16.504937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config ","metadata":{}},{"cell_type":"code","source":"MODEL_NAME = 'tf_efficientnet_b5_ns'\nBATCH_SIZE = 64\nIMAGE_SIZE = 900\nNUM_WORKERS = 15\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nUSE_AMP = True\nINIT = False\n\nroot_in = '../input/small-jpegs-fgvc' #Folder with input (image, lable)\nroot_out = './' #Folder with output (csv, pth) \nhave_index = False # If the breed label have been map to a index\n\n'''ArcFace parameter'''\nNUM_CLASSES = 100\nEMBEDDING_SIZE = 1024\nS, M = 30.0, 0.5 # S:consine scale in arcloss. M:arg penalty\nEASY_MERGING, LS_EPS = False, 0.0","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:34.708472Z","iopub.execute_input":"2022-04-21T02:22:34.708801Z","iopub.status.idle":"2022-04-21T02:22:34.715587Z","shell.execute_reply.started":"2022-04-21T02:22:34.708767Z","shell.execute_reply":"2022-04-21T02:22:34.714852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n    Args:\n        in_features: size of each input sample\n        out_features: size of each output sample\n        s: norm of input feature\n        m: margin\n        cos(theta + m)\n    \"\"\"\n\n    def __init__(\n        self,\n        in_features: int,\n        out_features: int,\n        s: float,\n        m: float,\n        easy_margin: bool,\n        ls_eps: float,\n        rank\n    ):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n        self.rank = rank\n\n    def forward(self, input: torch.Tensor, label: torch.Tensor, device = 'cuda') -> torch.Tensor:\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        # Enable 16 bit precision\n        cosine = cosine.to(torch.float32)\n\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=self.rank)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:23.41351Z","iopub.execute_input":"2022-04-21T02:22:23.413805Z","iopub.status.idle":"2022-04-21T02:22:23.430309Z","shell.execute_reply.started":"2022-04-21T02:22:23.413773Z","shell.execute_reply":"2022-04-21T02:22:23.429149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SorghumModel(nn.Module):\n    def __init__(self, model_name, embedding_size, map_location, k_fold, rank, pretrained=True):\n        super(SorghumModel, self).__init__()       \n        \n        #model_effecient_b6 = timm.create_model(model_name, pretrained=pretrained, num_classes=NUM_CLASSES)\n        #global param_name\n        #param_name = [name for name,_ in model_effecient_b6.named_parameters()] # All parameters name\n        #del model_effecient_b6\n            \n        self.model = timm.create_model(model_name, pretrained=pretrained, num_classes=NUM_CLASSES)\n        \n        #freeze_pretrained_layers(self.model)\n        #debarcle_layers(self.model, db_all=True) # Debarcle all layers()\n        \n        print('load Start!!!')\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.pooling = self.model.global_pool\n        self.model.global_pool = nn.Identity()\n        #self.pooling = GeM()\n        self.rank = rank\n        self.multiple_dropout = [nn.Dropout(0.25) for i in range(8)]\n        self.embedding = nn.Linear(in_features * 2, embedding_size)\n        self.fc = ArcMarginProduct(embedding_size, \n                                   NUM_CLASSES,\n                                   S, \n                                   M, \n                                   EASY_MERGING, \n                                   LS_EPS,\n                                  self.rank)\n\n    def forward(self, images, labels):\n        features = self.model(images)\n        pooled_features_avg = self.pooling(features).flatten(1)\n        pooled_features_max = nn.AdaptiveMaxPool2d((1,1))(features).flatten(1)\n        pooled_features = torch.cat((pooled_features_avg, pooled_features_max), dim=1)\n        pooled_features_dropout = torch.zeros((pooled_features.shape),device=self.rank)\n        for i in range(8):\n            pooled_features_dropout += self.multiple_dropout[i](pooled_features)\n        pooled_features_dropout /= 8\n        embedding = self.embedding(pooled_features_dropout)\n        #pooled_features = nn.Dropout(0.5)(pooled_features)\n        #embedding = self.embedding(pooled_features)\n        output = self.fc(embedding, labels)\n        return output\n    \n    def extract(self, images):\n        features = self.model(images)\n        pooled_features_avg = self.pooling(features).flatten(1)\n        pooled_features_max = nn.AdaptiveMaxPool2d((1,1))(features).flatten(1)\n        pooled_features = torch.cat((pooled_features_avg, pooled_features_max), dim=1)\n        embedding = self.embedding(pooled_features)\n        return embedding","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:23.869652Z","iopub.execute_input":"2022-04-21T02:22:23.869957Z","iopub.status.idle":"2022-04-21T02:22:23.885336Z","shell.execute_reply.started":"2022-04-21T02:22:23.869922Z","shell.execute_reply":"2022-04-21T02:22:23.88429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"tags":[]}},{"cell_type":"code","source":"class Sorghum_Train_Dataset(Dataset):\n    '''Train Dataset'''\n    def __init__(self, img_path_csv='', df=None, transform=None):\n        if df is not None:\n            self.df = df\n        else:\n            self.df = pd.read_csv(img_path_csv)\n        self.transform = transform\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index):\n        img = Img.open(os.path.join(root_in, 'train', self.df.iloc[index, 0]))\n        label_index = self.df.iloc[index, 4]\n        if self.transform is not None:\n            img = self.transform(img)\n        return img, label_index","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:25.582431Z","iopub.execute_input":"2022-04-21T02:22:25.582978Z","iopub.status.idle":"2022-04-21T02:22:25.590987Z","shell.execute_reply.started":"2022-04-21T02:22:25.582927Z","shell.execute_reply":"2022-04-21T02:22:25.590048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Sorghum_Test_Dataset(Sorghum_Train_Dataset):\n    '''Test Dataset'''\n    def __getitem__(self, index):\n        img = Img.open(os.path.join(root_in, 'test', self.df.iloc[index, 0]))\n        if self.transform:\n            img = self.transform(img)\n        return img","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:22:25.72826Z","iopub.execute_input":"2022-04-21T02:22:25.728561Z","iopub.status.idle":"2022-04-21T02:22:25.734053Z","shell.execute_reply.started":"2022-04-21T02:22:25.728529Z","shell.execute_reply":"2022-04-21T02:22:25.733324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data-pre-process","metadata":{}},{"cell_type":"code","source":"def data_pre_access(file, output):\n    '''transfer train label into index'''\n    labels = pd.read_csv(file, index_col='image')\n    labels_map = dict()\n    labels['label_index'] = torch.zeros((labels.shape[0])).type(torch.int32).numpy()\n    for i, label in enumerate(labels.cultivar.unique()):\n        labels_map[i] = label\n        labels.loc[labels.cultivar == label, 'label_index'] = i\n    labels.to_csv(output)\n    \n    return labels_map","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:33:12.460968Z","iopub.execute_input":"2022-04-21T02:33:12.461884Z","iopub.status.idle":"2022-04-21T02:33:12.469763Z","shell.execute_reply.started":"2022-04-21T02:33:12.461834Z","shell.execute_reply":"2022-04-21T02:33:12.468931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check labels_map(map from index to cultivar)","metadata":{"tags":[]}},{"cell_type":"markdown","source":"##### Here, if you have labels_index.csv, please turn the have_index=True","metadata":{}},{"cell_type":"code","source":"if have_index:\n    labels_map = {}\n    train_df = pd.read_csv(os.path.join(root_out,'labels_index.csv'), index_col='image')\n    def label_f(m):\n        labels_map[int(m.label_index)] = m.cultivar\n    train_df.apply(label_f,axis=1)\nelse:\n    labels_map = data_pre_access(os.path.join(root_in,'train_cultivar_mapping.csv'), output=os.path.join(root_out,'labels_index.csv'))\n    train_df = pd.read_csv(os.path.join(root_out,'labels_index.csv'), index_col='image')\nnum_classes = len(labels_map)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:34:40.041506Z","iopub.execute_input":"2022-04-21T02:34:40.0418Z","iopub.status.idle":"2022-04-21T02:34:40.611813Z","shell.execute_reply.started":"2022-04-21T02:34:40.04177Z","shell.execute_reply":"2022-04-21T02:34:40.610789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:34:42.459612Z","iopub.execute_input":"2022-04-21T02:34:42.459887Z","iopub.status.idle":"2022-04-21T02:34:42.46697Z","shell.execute_reply.started":"2022-04-21T02:34:42.459856Z","shell.execute_reply":"2022-04-21T02:34:42.465781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_sum = 0\nfor key, val in tqdm(labels_map.items()):\n    train_df[train_df.label_index == key].cultivar.unique() == val\n    check_sum += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:34:43.334963Z","iopub.execute_input":"2022-04-21T02:34:43.335576Z","iopub.status.idle":"2022-04-21T02:34:43.431439Z","shell.execute_reply.started":"2022-04-21T02:34:43.33553Z","shell.execute_reply":"2022-04-21T02:34:43.430449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_sum == len(labels_map)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:34:44.860663Z","iopub.execute_input":"2022-04-21T02:34:44.860952Z","iopub.status.idle":"2022-04-21T02:34:44.866961Z","shell.execute_reply.started":"2022-04-21T02:34:44.860921Z","shell.execute_reply":"2022-04-21T02:34:44.866351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load model","metadata":{}},{"cell_type":"markdown","source":"##### Don't foget enable code below when you inference","metadata":{}},{"cell_type":"code","source":"#model_dict = torch.load(os.path.join(root_out,'tf_efficientnet_b5_ns_F_0/Sorghum17.params'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nmodel_dict_noDist = {}\nfor key, value in model_dict.items():\n    model_dict_noDist[key.split('module.')[-1]] = model_dict[key]\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nmodel = SorghumModel(MODEL_NAME, EMBEDDING_SIZE, map_location={'cuda:%d' % 0: 'cuda:%d' % 0}, k_fold=0, rank=0)\nmodel.load_state_dict(model_dict_noDist)\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Here we predict all the values(acc@all), instead of max values index(acc@1)","metadata":{}},{"cell_type":"code","source":"def predict_test_raw(net, test_iter, device=None):\n    '''Inference'''\n    net.eval()\n    if isinstance(net, nn.Module):\n        net.eval()\n        if not device:\n            device = next(iter(net.parameters())).device\n    y = []\n    net.to(device)\n    softmax = nn.Softmax(dim=1)\n    with torch.no_grad():\n        for X in tqdm(test_iter):\n            if isinstance(X, list):\n                X = [x.to(device) for x in X]\n            else:\n                X = X.to(device)\n            with torch.cuda.amp.autocast(enabled=True):\n                embeddings = net.extract(X)\n                y += softmax(S * F.linear(F.normalize(embeddings), F.normalize(net.fc.weight))).cpu()\n    return np.array(list(Y.numpy() for Y in y))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save inference result","metadata":{"tags":[]}},{"cell_type":"code","source":"def CLAHE_Convert(origin_input):\n    clahe = cv.createCLAHE(clipLimit=40, tileGridSize=(10,10))\n    t = np.asarray(origin_input)\n    t = cv.cvtColor(t, cv.COLOR_BGR2HSV)\n    t[:,:,-1] = clahe.apply(t[:,:,-1])\n    t = cv.cvtColor(t, cv.COLOR_HSV2BGR)\n    t = Img.fromarray(t)\n    return t","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:38:22.183554Z","iopub.execute_input":"2022-04-21T02:38:22.183982Z","iopub.status.idle":"2022-04-21T02:38:22.18994Z","shell.execute_reply.started":"2022-04-21T02:38:22.183952Z","shell.execute_reply":"2022-04-21T02:38:22.189099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    CLAHE_Convert,\n    transforms.Resize(IMAGE_SIZE),\n    transforms.ColorJitter(brightness=0.2, contrast=0.05, saturation=0.1),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.RandomApply(transforms=\n                  [transforms.RandomResizedCrop(size=IMAGE_SIZE, scale=(0.3,0.4), \n                                                ratio=(1/3,3),interpolation=\n                                                transforms.InterpolationMode.BICUBIC)],p=0.2),\n    transforms.ToTensor(),\n    # Normalize to fit pretrained model\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])\n\nval_test_transform = transforms.Compose([\n    CLAHE_Convert,\n    transforms.Resize(IMAGE_SIZE),\n    transforms.ToTensor(),\n    # Normalize to fit pretrained model\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])","metadata":{"execution":{"iopub.status.busy":"2022-04-21T02:38:23.056739Z","iopub.execute_input":"2022-04-21T02:38:23.057084Z","iopub.status.idle":"2022-04-21T02:38:23.066358Z","shell.execute_reply.started":"2022-04-21T02:38:23.05705Z","shell.execute_reply":"2022-04-21T02:38:23.065287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorghum_test_dataset = Sorghum_Test_Dataset('../input/sorghum-jpeg-test-csv/test.csv', transform=val_test_transform)\nsorghum_test_loader = DataLoader(sorghum_test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:13:08.16138Z","iopub.execute_input":"2022-04-21T03:13:08.163781Z","iopub.status.idle":"2022-04-21T03:13:08.217819Z","shell.execute_reply.started":"2022-04-21T03:13:08.163668Z","shell.execute_reply":"2022-04-21T03:13:08.216604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Enable code below when you inference'''\n#result_raw_original = predict_test_raw(model, sorghum_test_loader, device='cuda')\n#np.save(os.path.join(root_out, 'test_result_raw_Original.npy'), result_raw_original)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:14:20.007036Z","iopub.execute_input":"2022-04-21T03:14:20.007327Z","iopub.status.idle":"2022-04-21T03:14:20.013855Z","shell.execute_reply.started":"2022-04-21T03:14:20.007295Z","shell.execute_reply":"2022-04-21T03:14:20.013218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TTA","metadata":{"tags":[]}},{"cell_type":"markdown","source":"## TTA Transforms","metadata":{"tags":[]}},{"cell_type":"code","source":"tta_transform0 = transforms.Compose([\n    CLAHE_Convert,\n    transforms.Resize(IMAGE_SIZE),\n    transforms.RandomAffine(degrees=(0, 45), translate=(0.05, 0.1), scale=(0.95, 1)),\n    transforms.ToTensor(),\n    # Normalize to fit pretrained model\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])\n\ntta_transform1 = transforms.Compose([\n    CLAHE_Convert,\n    transforms.Resize(IMAGE_SIZE),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.ColorJitter(brightness=0.2, contrast=0.01, saturation=0.2),\n    transforms.ToTensor(),\n    # Normalize to fit pretrained model\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])\n\ntta_transform2 = transforms.Compose([\n    CLAHE_Convert,\n    transforms.Resize(IMAGE_SIZE),\n    transforms.RandomApply(transforms=\n              [transforms.RandomResizedCrop(size=IMAGE_SIZE, scale=(0.4,0.5), \n                                            ratio=(1/3,3),interpolation=\n                                            transforms.InterpolationMode.BICUBIC)],p=0.2),\n    transforms.ToTensor(),\n    # Normalize to fit pretrained model\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tta_transforms = [tta_transform0,\n                  tta_transform1,\n                  tta_transform2]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save TTA result","metadata":{"tags":[]}},{"cell_type":"code","source":"'''Enable the last line when you inference.\n    The first line is just used to show result'''\nresult_raw_original = np.load('../input/sorghum-jpeg-test-csv/test_result_raw_Original.npy')\n#result_raw_original = np.load(os.path.join(root_out, 'test_result_raw_Original.npy'))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:17:08.352861Z","iopub.execute_input":"2022-04-21T03:17:08.353204Z","iopub.status.idle":"2022-04-21T03:17:08.573976Z","shell.execute_reply.started":"2022-04-21T03:17:08.353171Z","shell.execute_reply":"2022-04-21T03:17:08.573233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_raw_ttas = {'origin':result_raw_original, 'avg':result_raw_original}","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:17:10.857474Z","iopub.execute_input":"2022-04-21T03:17:10.858121Z","iopub.status.idle":"2022-04-21T03:17:10.86511Z","shell.execute_reply.started":"2022-04-21T03:17:10.858087Z","shell.execute_reply":"2022-04-21T03:17:10.864384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Enable code below when you inference\n'''for i in range(len(tta_transforms)):\n    torch.cuda.empty_cache()\n    sorghum_test_dataset = Sorghum_Test_Dataset(os.path.join(root_in, 'test.csv'), transform=tta_transforms[i])\n    sorghum_test_loader = DataLoader(sorghum_test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)\n    result_raw_tta = predict_test_raw(model, sorghum_test_loader, device='cuda')\n    np.save(os.path.join(root_out, 'test_result_raw_' + 'tta_' + str(i) + '_.npy'), result_raw_tta)\n    result_raw_ttas['tta_' + str(i)] = result_raw_tta\n    result_raw_ttas['avg'] += result_raw_tta'''","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_raw_ttas['avg'] /= len(result_raw_ttas.keys()) - 1","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:18:16.366316Z","iopub.execute_input":"2022-04-21T03:18:16.366779Z","iopub.status.idle":"2022-04-21T03:18:16.373416Z","shell.execute_reply.started":"2022-04-21T03:18:16.366742Z","shell.execute_reply":"2022-04-21T03:18:16.372428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inspect the result","metadata":{"tags":[]}},{"cell_type":"markdown","source":"## Sort the result","metadata":{"tags":[]}},{"cell_type":"code","source":"result_ttas_sorted_val = {}\nresult_ttas_sorted_idx = {}","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:18:37.682301Z","iopub.execute_input":"2022-04-21T03:18:37.682927Z","iopub.status.idle":"2022-04-21T03:18:37.687858Z","shell.execute_reply.started":"2022-04-21T03:18:37.682864Z","shell.execute_reply":"2022-04-21T03:18:37.687062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_raw_ttas.keys()","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:18:38.939108Z","iopub.execute_input":"2022-04-21T03:18:38.93939Z","iopub.status.idle":"2022-04-21T03:18:38.945845Z","shell.execute_reply.started":"2022-04-21T03:18:38.93936Z","shell.execute_reply":"2022-04-21T03:18:38.945037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key, val in result_raw_ttas.items():\n    torch.cuda.empty_cache()\n    result_tta = torch.tensor(val, dtype=torch.float32, device='cuda')\n    result_sorted_val, result_sorted_idx = result_tta.sort(dim=1,descending=True)\n    result_ttas_sorted_val[key] = result_sorted_val.cpu().numpy()\n    result_ttas_sorted_idx[key] = result_sorted_idx.cpu().numpy()\n    del result_tta, result_sorted_val, result_sorted_idx","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize","metadata":{"tags":[]}},{"cell_type":"code","source":"sns.displot(result_ttas_sorted_val.get('origin')[:,[0,1,2]], kind='ecdf')","metadata":{"execution":{"iopub.status.busy":"2022-04-21T03:19:12.028418Z","iopub.execute_input":"2022-04-21T03:19:12.028926Z","iopub.status.idle":"2022-04-21T03:19:12.066013Z","shell.execute_reply.started":"2022-04-21T03:19:12.028876Z","shell.execute_reply":"2022-04-21T03:19:12.06476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(result_ttas_sorted_val.get('avg')[:,[0,1,2]], kind='ecdf')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_original_df = pd.DataFrame(result_ttas_sorted_val['origin'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_original_df.iloc[:,[0,1]].describe(percentiles=[0.05, 0.25, 0.35, 0.45, 0.65, 0.75, 0.95])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_tta_avg_df = pd.DataFrame(result_ttas_sorted_val['avg'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_tta_avg_df.iloc[:,[0,1]].describe(percentiles=[0.05, 0.25, 0.35, 0.45, 0.65, 0.75, 0.95])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Find the trust Threshold (no use in this competition)","metadata":{"tags":[]}},{"cell_type":"code","source":"Threshold = result_tta_avg_df.iloc[:,0].mean() - result_tta_avg_df.iloc[:,0].std() / 2 # trust interval [μ - σ/2, )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Threshold","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict and make submission","metadata":{"tags":[]}},{"cell_type":"code","source":"result_ttas_sorted_val['avg'].shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_sorted_val = result_ttas_sorted_val['avg']\nresult_sorted_idx = result_ttas_sorted_idx['avg']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(result_sorted_val)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_file =  pd.read_csv(os.path.join(root_in, 'sample_submission.csv'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''check the order between test.csv.image and sample_submission.csv.filename'''\nresult = pd.read_csv('../input/sorghum-jpeg-test-csv/test.csv')\nsum(result.image.map(lambda x:x.split('.jpeg')[0]) == sub_file.filename.map(lambda x:x.split('.png')[0]))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.read_csv('../input/small-jpegs-fgvc/sample_submission.csv')\nresult['cultivar'] = [labels_map.get(result_sorted_idx[i,0]) for i in range(result_sorted_idx.shape[0])]\nresult = result.set_index('filename')\nresult.to_csv(os.path.join(root_out, 'submission.csv'))","metadata":{},"execution_count":null,"outputs":[]}]}