{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport zipfile\nfrom copy import deepcopy\nimport random\nimport math\nimport shutil\nimport gc as garbage\nimport collections\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchmetrics import functional as FM\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\n\nimport PIL\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport seaborn as sns\n\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm as tqdm_nb\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import preprocessing\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-24T16:19:07.151745Z","iopub.execute_input":"2022-04-24T16:19:07.152149Z","iopub.status.idle":"2022-04-24T16:19:14.32887Z","shell.execute_reply.started":"2022-04-24T16:19:07.152097Z","shell.execute_reply":"2022-04-24T16:19:14.328082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"def get_pil_image(dirname, path):\n    return PIL.Image.open(os.path.join(f'../input/{dirname}', path))\n\ndef get_image(dirname, path):\n    return np.array(get_pil_image(dirname, path))\n\ndef image_histplot(img, **kwargs):\n    if not 'figsize' in kwargs: kwargs['figsize'] = (12, 3)\n    fig, axes = plt.subplots(1, 4, **kwargs)\n    plt.tight_layout()\n    axes[0].imshow(img)\n    for i in range(3):\n        axes[1+i].set_yticks([])\n        sns.histplot(img[:,:,i].flatten() / 255, ax=axes[1+i], color=\"rgb\"[i], alpha=0.33)\n    plt.show()\n\ndef image_gridplot(images, rows=None, cols=None, transform=None, **kwagrs):\n    if rows == None and cols == None:\n        cols = 6\n        rows = math.ceil(len(images) / cols)\n    elif rows == None:\n        rows = math.ceil(len(images) / cols)\n    elif cols == None:\n        cols = math.ceil(len(images) / rows)\n    if not 'figsize' in kwagrs: kwagrs['figsize'] = (3 * cols, 3 * rows)\n    fig = plt.figure(**kwagrs)\n    grid = ImageGrid(fig, 111, nrows_ncols=(rows, cols), axes_pad=0.1)\n    for ax, im in zip(grid, images):\n        ax.axis('off')\n        if transform != None: im = transform(im)\n        ax.imshow(im)\n    plt.show()\n\nclass ImageConv:\n    def tensor2np(tensor):\n        return tensor.permute(1, 2, 0).detach().cpu().numpy().astype(np.uint8)\n\n    def np2tensor(nparr):\n#         return torch.tensor(nparr).to(Hypers.device).permute(2, 0, 1).float()\n        return torch.tensor(nparr).permute(2, 0, 1).float()\n    \n    def to_plt(x):\n        if type(x) == torch.Tensor:\n            return ImageConv.tensor2np(x).astype(np.uint8)\n        return x.astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:14.330755Z","iopub.execute_input":"2022-04-24T16:19:14.331004Z","iopub.status.idle":"2022-04-24T16:19:14.344563Z","shell.execute_reply.started":"2022-04-24T16:19:14.33097Z","shell.execute_reply":"2022-04-24T16:19:14.343895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class Hypers:\n    verbose = True\n    \n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    image_size = (224, 224)\n    \n    max_epoches = 80\n    batch_size = 128\n    learning_rate = 1e-3\n    optimizer = torch.optim.Adam\n    \n    patience = 5\n\n\n# CUDA set-up\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:14.345842Z","iopub.execute_input":"2022-04-24T16:19:14.346241Z","iopub.status.idle":"2022-04-24T16:19:14.419799Z","shell.execute_reply.started":"2022-04-24T16:19:14.346203Z","shell.execute_reply":"2022-04-24T16:19:14.418811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv')\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:14.421928Z","iopub.execute_input":"2022-04-24T16:19:14.422333Z","iopub.status.idle":"2022-04-24T16:19:14.480063Z","shell.execute_reply.started":"2022-04-24T16:19:14.422293Z","shell.execute_reply":"2022-04-24T16:19:14.479352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution","metadata":{}},{"cell_type":"code","source":"image_count_df = df_train.groupby('cultivar').count().sort_values('image')\nprint('Skew: ', image_count_df.skew())\n\nplt.figure(figsize=(20, 8))\nplt.xticks(rotation=90)\nsns.barplot(x=image_count_df.index, y=image_count_df['image'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:14.481424Z","iopub.execute_input":"2022-04-24T16:19:14.48166Z","iopub.status.idle":"2022-04-24T16:19:15.987244Z","shell.execute_reply.started":"2022-04-24T16:19:14.481626Z","shell.execute_reply":"2022-04-24T16:19:15.986539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Color Histogram","metadata":{}},{"cell_type":"code","source":"for i in range(3):\n    image_path, cultivar = df_train.loc[i]\n    image_histplot(get_image('sorghum-id-fgvc-9/train_images', image_path), figsize=(15, 3))","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:15.988473Z","iopub.execute_input":"2022-04-24T16:19:15.989562Z","iopub.status.idle":"2022-04-24T16:19:29.782978Z","shell.execute_reply.started":"2022-04-24T16:19:15.989513Z","shell.execute_reply":"2022-04-24T16:19:29.782288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NOTE:** First, it contains dummy data `.DS_STORE`","metadata":{}},{"cell_type":"code","source":"print('train set has', df_train.isna().sum().max(), 'null(s)')","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:29.784073Z","iopub.execute_input":"2022-04-24T16:19:29.785923Z","iopub.status.idle":"2022-04-24T16:19:29.796821Z","shell.execute_reply.started":"2022-04-24T16:19:29.785882Z","shell.execute_reply":"2022-04-24T16:19:29.795953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outliers","metadata":{}},{"cell_type":"code","source":"outliers = ['29-33-477', '29-34-965', '29-36-468', '29-43-957', '29-45-460', '29-46-961', '29-48-469', '29-49-960', '29-51-465', '30-06-475', '30-07-971', '30-09-467', '30-59-251', '31-00-751', '31-02-238', '31-17-229', '31-18-730', '31-20-230', '31-21-751', '31-23-234', '31-24-729', '31-30-733', '31-32-233', '31-33-750', '31-35-254']\noutliers = list(map(lambda id: f'2017-06-11__13-{id}.png', outliers))\noutliers_img = list(map(lambda id: np.array(get_pil_image('sorghum-id-fgvc-9/train_images', id)), outliers))\nimage_gridplot(outliers_img, cols=10, figsize=(12, 3))","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:29.798118Z","iopub.execute_input":"2022-04-24T16:19:29.798383Z","iopub.status.idle":"2022-04-24T16:19:35.159862Z","shell.execute_reply.started":"2022-04-24T16:19:29.798347Z","shell.execute_reply":"2022-04-24T16:19:35.159222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.drop(df_train[df_train['image'].isin(outliers)].index)\ndf_train = df_train.dropna().reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:35.161183Z","iopub.execute_input":"2022-04-24T16:19:35.161636Z","iopub.status.idle":"2022-04-24T16:19:35.180527Z","shell.execute_reply.started":"2022-04-24T16:19:35.1616Z","shell.execute_reply":"2022-04-24T16:19:35.179894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# difficult cases\n\njust in my opinion","metadata":{}},{"cell_type":"code","source":"hard_pathes = [\n    ('sorghum-id-fgvc-9/train_images', '2017-06-02__13-44-49-948.png'),\n    ('sorghum-id-fgvc-9/train_images', '2017-06-02__16-46-57-375.png'),\n    ('sorghum-id-fgvc-9/test', '181578.png'),\n    ('sorghum-id-fgvc-9/test', '13937931.png'),\n    ('sorghum-id-fgvc-9/test', '17713602.png'),\n    ('sorghum-id-fgvc-9/test', '21517068.png'),\n    ('sorghum-id-fgvc-9/test', '45744669.png'),\n    ('sorghum-id-fgvc-9/test', '326836917.png'),\n    ('sorghum-id-fgvc-9/test', '1368455764.png'),\n    ('sorghum-id-fgvc-9/test', '1119726178.png'),\n]\nhard_imgs = list(map(lambda case: get_image(case[0], case[1]), hard_pathes))\nimage_gridplot(hard_imgs, cols=5)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:35.183603Z","iopub.execute_input":"2022-04-24T16:19:35.183793Z","iopub.status.idle":"2022-04-24T16:19:37.522936Z","shell.execute_reply.started":"2022-04-24T16:19:35.183768Z","shell.execute_reply":"2022-04-24T16:19:37.522318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CLAHE (Contrast Limited Adaptive Histogram Equalization)","metadata":{}},{"cell_type":"code","source":"import cv2\n\ndef CLAHE(image):\n    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n    clahe = cv2.createCLAHE(clipLimit=10)\n    hsv[:,:,-1] = clahe.apply(hsv[:,:,-1])\n    rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)\n    return rgb","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:37.524213Z","iopub.execute_input":"2022-04-24T16:19:37.52464Z","iopub.status.idle":"2022-04-24T16:19:37.673502Z","shell.execute_reply.started":"2022-04-24T16:19:37.524604Z","shell.execute_reply":"2022-04-24T16:19:37.672766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = []\nfor i in [0, 20, 40]:\n    img_path, c = df_train.loc[i]\n    img = get_image('sorghum-id-fgvc-9/train_images', img_path)\n    images.append(img)\n    images.append(CLAHE(img))\n\nimage_gridplot(images, cols=6)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:37.67604Z","iopub.execute_input":"2022-04-24T16:19:37.676533Z","iopub.status.idle":"2022-04-24T16:19:38.863862Z","shell.execute_reply.started":"2022-04-24T16:19:37.676492Z","shell.execute_reply":"2022-04-24T16:19:38.859749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_gridplot(list(map(CLAHE, hard_imgs)), cols=5)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:38.865267Z","iopub.execute_input":"2022-04-24T16:19:38.86572Z","iopub.status.idle":"2022-04-24T16:19:40.61094Z","shell.execute_reply.started":"2022-04-24T16:19:38.865684Z","shell.execute_reply":"2022-04-24T16:19:40.610285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_histplot(images[-2])\nimage_histplot(images[-1])","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:40.611968Z","iopub.execute_input":"2022-04-24T16:19:40.612327Z","iopub.status.idle":"2022-04-24T16:19:48.306928Z","shell.execute_reply.started":"2022-04-24T16:19:40.612295Z","shell.execute_reply":"2022-04-24T16:19:48.306257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Good.","metadata":{}},{"cell_type":"markdown","source":"# DataLoader","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nclass SorghumDataset(Dataset): \n    def __init__(self, df, is_train=True, transform=None, encoder=LabelEncoder()):\n        super().__init__()\n        \n        self.is_train = is_train\n        self.df = df\n        self.transform = transform\n        self.encoder = encoder\n        if self.is_train:\n            self.df['label'] = self.encoder.fit_transform(self.df['cultivar'])\n\n    @property\n    def data(self):\n        return self.df.values\n    \n    @property\n    def targets(self):\n        return self.df['label']\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if self.is_train:\n            image_path, cultivar, label = self.df.loc[idx, ['image', 'cultivar', 'label']]\n        else:\n            image_path, cultivar, label = self.df.loc[idx, ['filename', 'cultivar', 'filename']]\n        dataset_name = 'sorghum-fgvc9-clahe-256' # 'sorghum-id-fgvc-9'\n        image = get_image(f'{dataset_name}/train_images' if self.is_train else f'{dataset_name}/test', image_path)\n        return self.transform(image), cultivar, label","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:48.308339Z","iopub.execute_input":"2022-04-24T16:19:48.308816Z","iopub.status.idle":"2022-04-24T16:19:48.318905Z","shell.execute_reply.started":"2022-04-24T16:19:48.308781Z","shell.execute_reply":"2022-04-24T16:19:48.318174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SorghumDataset(df_train, is_train=True, transform=np.array).df.head(15)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:48.32003Z","iopub.execute_input":"2022-04-24T16:19:48.320374Z","iopub.status.idle":"2022-04-24T16:19:48.341003Z","shell.execute_reply.started":"2022-04-24T16:19:48.320338Z","shell.execute_reply":"2022-04-24T16:19:48.340372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforming","metadata":{}},{"cell_type":"code","source":"%%time\nfrom torchvision import transforms as T\n\nmean = torch.tensor([0.485, 0.456, 0.406], dtype=torch.float32)\nstd = torch.tensor([0.229, 0.224, 0.225], dtype=torch.float32)\nnormalize = T.Normalize(mean.tolist(), std.tolist())\nunnormalize = T.Normalize((-mean / std).tolist(), (1.0 / std).tolist())\n\ntransform_train = T.Compose([\n    ImageConv.np2tensor,\n    T.Resize(512),\n    T.RandomRotation(45, fill=(82, 83, 65)),\n    T.CenterCrop(384),\n    T.RandomCrop(320),\n    T.Resize(Hypers.image_size),\n    normalize,\n])\n\ndataset = SorghumDataset(df_train, is_train=True, transform=transform_train)\ndataloader = DataLoader(dataset, batch_size=16)\n\nimages, _, labels = next(iter(dataloader))\nimage_gridplot(images[:12], transform=T.Compose([\n    ImageConv.to_plt,\n]))\nimage_gridplot(images[:12], transform=T.Compose([\n    unnormalize,\n    ImageConv.to_plt,\n]))\n\ndel dataset\ndel dataloader\ndel images\ndel labels","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:49:44.187853Z","iopub.execute_input":"2022-04-24T16:49:44.188117Z","iopub.status.idle":"2022-04-24T16:49:46.902038Z","shell.execute_reply.started":"2022-04-24T16:49:44.188088Z","shell.execute_reply":"2022-04-24T16:49:46.90135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Model","metadata":{}},{"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        # output = (input - kernel + 2*padding) / stride + 1\n\n        self.layer = nn.Sequential(\n            nn.Conv2d(3, 16, 3, padding=1),    # (3, 128, 128) -> (16, 128, 128)\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n            nn.Conv2d(16, 128, 5, stride=3),   # (16, 128, 128) -> (128, 42, 42)\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(2, stride=1),         # (128, 42, 42) -> (128, 40, 40)\n\n            nn.Conv2d(128, 256, 3),            # (128, 40, 40) -> (256, 38, 38)\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.Conv2d(256, 256, 3),            # (128, 38, 38) -> (256, 36, 36)\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2),                # (256, 36, 36) -> (256, 18, 18)\n\n            nn.Conv2d(256, 512, 3),            # (256, 20, 20) -> (512, 16, 16)\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2),                # (512, 16, 16) -> (512, 8, 8)\n        )\n        self.classifier = nn.Sequential(\n            nn.Linear(512 * 8 * 8, 1024),\n            nn.ReLU(),\n#             nn.Dropout(0.2),\n            nn.Linear(1024, 100),\n        )\n    \n    def forward(self, x):\n        x = self.layer(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:19:52.282944Z","iopub.execute_input":"2022-04-24T16:19:52.283388Z","iopub.status.idle":"2022-04-24T16:19:52.296995Z","shell.execute_reply.started":"2022-04-24T16:19:52.283352Z","shell.execute_reply":"2022-04-24T16:19:52.296274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Trainer","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\n\nclass Trainer:\n    def __init__(\n        self,\n        loss_function,\n        optimizer,\n        max_epochs,\n        accelerator='cpu',\n        min_epochs=1,\n        progress=False,\n        batch_parser=None,\n        logging=True,\n        verbose=False,\n        callbacks=[],\n        **kwargs\n    ):\n        self.log = {}\n        self.criterion = loss_function\n        self.optimizer = optimizer\n        self.device = accelerator\n        self.min_epochs = min_epochs\n        self.max_epochs = max_epochs\n        self.has_progress = progress\n        self.has_logging = logging\n        self.verbose = verbose\n        self.pbar = None\n        self.batch_parser = batch_parser if batch_parser else Trainer.default_parser\n        self.start_time = datetime.now()\n        self.callbacks = callbacks\n\n    def default_parser(batch):\n        x, y = batch\n        return x, y\n    \n    def set_parser(self, function):\n        self.batch_parser = function\n        \n    def logging(self, name, value):\n        if not self.has_logging: pass\n        if not name in self.log:\n            self.log[name] = []\n        self.log[name].append(value)\n\n    def step_loop(self, model, dataloader, is_train=True):\n        torch.cuda.synchronize()\n        model.to(self.device)\n        if is_train:\n            model.train()\n        else:\n            model.eval()\n        loss_sum = 0\n        acc_sum = 0\n        counter = 0\n        for batch in dataloader:\n            counter += 1\n            x, y = self.batch_parser(batch)\n            x, y = x.to(self.device), y.to(self.device)\n            y_hat = model(x).to(self.device)\n            loss = self.criterion(y_hat, y)\n            _, y_pred = torch.max(y_hat.data, 1)\n            acc = (y_pred == y).sum()\n            loss_sum += loss.item()\n            acc_sum += acc.item()\n            if is_train:\n                self.optimizer.zero_grad()\n                loss.backward()\n                self.optimizer.step()\n            if type(None) != type(self.pbar):\n                self.pbar.update(1)\n        loss_sum /= counter\n        acc_sum /= counter\n        return loss_sum, acc_sum\n    \n    def train(self, model, dataloader):\n        return self.step_loop(model, dataloader, True)\n    \n    def valid(self, model, dataloader):\n        return self.step_loop(model, dataloader, True)\n    \n    def test(self, model, dataloader):\n        return self.step_loop(model, dataloader, False)\n\n    def fit(self, model, load_dataloaders):\n        if self.has_progress:\n            self.progress_start().set_description('on ready to fit')\n        self.start_time = datetime.now()\n        \n        # torch.cuda.empty_cache()\n        garbage.collect()\n        \n        for cb in self.callbacks:\n            if issubclass(type(cb), CheckpointCallback):\n                cb.before_fit(model)\n\n        for e in range(self.max_epochs):\n            train_loader, valid_loader = load_dataloaders()\n        \n            if self.has_progress:\n                self.pbar.reset(len(train_loader) + len(valid_loader))\n                self.pbar.set_description(f'[epoch={e+1}/{self.max_epochs}] train')\n                \n            train_loss, train_acc = self.train(model, train_loader)\n            self.logging('train_loss', train_loss)\n            self.logging('train_acc', train_acc)\n            \n            if self.has_progress:\n                self.pbar.set_description(f'[epoch={e+1}/{self.max_epochs}] valid')\n                \n            valid_loss, valid_acc = self.valid(model, valid_loader)\n            self.logging('valid_loss', valid_loss)\n            self.logging('valid_acc', valid_acc)\n            \n            time_elapsed = (datetime.now() - self.start_time)\n            msg = 'train/valid loss: {:.05f}/{:.05f}'.format(train_loss, valid_loss)\n            \n            if self.has_progress: \n                self.pbar.set_postfix_str(msg)\n            \n            if self.verbose:\n                print(f'[Trainer(epoch={e+1})]', str(time_elapsed), msg)\n\n            early_stop = False\n            for cb in self.callbacks:\n                if issubclass(type(cb), CheckpointCallback):\n                    cb.on_train_end(model, np.mean([train_loss, valid_loss]))\n                elif issubclass(type(cb), EarlyStopCallback):\n                    early_stop |= cb.on_train_end(e, time_elapsed.seconds, self.log)\n                    if early_stop: break\n            \n            if e < self.min_epochs:\n                continue\n\n            if early_stop:\n                break\n                \n            garbage.collect()\n\n        self.progress_end()\n    \n    def progress_start(self, total=None, desc=None):\n        if type(None) == type(self.pbar):\n            self.pbar = tqdm_nb()\n        self.pbar.reset(total)\n        self.pbar.set_description(desc)\n        return self.pbar\n\n    def progress_end(self):\n        self.pbar.close()\n        self.pbar = None","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:49:52.263024Z","iopub.execute_input":"2022-04-24T16:49:52.263494Z","iopub.status.idle":"2022-04-24T16:49:52.290536Z","shell.execute_reply.started":"2022-04-24T16:49:52.263457Z","shell.execute_reply":"2022-04-24T16:49:52.289524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Callback class","metadata":{}},{"cell_type":"code","source":"class Callback:\n    def __init__(self, verbose=False):\n        self.verbose = verbose\n    def before_fit(self):\n        pass\n    def message(self, msg, prefix=''):\n        if self.verbose:\n            print(f'[{self.__class__.__name__}{prefix}] {msg}')\n    def on_train_end(self, epoch: int, time_elapsed: int):\n        pass\n\nclass EarlyStopCallback(Callback):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n    def on_train_end(self, epoch: int, time_elapsed: int, info: dict):\n        pass\n\nclass CheckpointCallback(Callback):\n    def __init__(\n        self,\n        name,\n        path='./',\n        metric=np.less,\n        min_delta=0.0,\n        load_from_checkpoint=False, # str: filepath\n        **kwargs\n    ):\n        super().__init__(**kwargs)\n        self.name = name\n        self.path = path\n        self.filepath = os.path.join(self.path, self.name)\n        self.min_delta = min_delta\n        self.metric = metric\n        self.load_from_checkpoint = load_from_checkpoint\n        self.best = np.Inf\n    \n    def load_model(self, model: nn.Module, filepath: str):\n        file_exists = os.path.exists(filepath)\n        if self.verbose:\n            if file_exists: self.message(f'model loaded from {self.filepath}')\n            else: self.message(f'Failed to load model from {self.filepath}')\n        if file_exists:\n            model.load_state_dict(torch.load(self.filepath))\n\n    def before_fit(self, model: nn.Module):\n        lfc = self.load_from_checkpoint\n        if isinstance(lfc, bool):\n            if lfc == True:\n                self.load_model(model, self.filepath)\n        elif isinstance(lfc, str):\n            self.load_model(model, lfc)\n        else:\n            self.message('Disallowed checkpoint parameter:', type(lfc), lfc)\n\n    # update and save model if loss is less than before\n    def on_train_end(self, model: nn.Module, loss: float):\n        if self.metric(self.best, loss + self.min_delta): return None\n        self.message(f'update best: {loss}', prefix=f'({self.filepath})')\n        self.best = loss\n        torch.save(model.state_dict(), self.filepath)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:49:52.729331Z","iopub.execute_input":"2022-04-24T16:49:52.729879Z","iopub.status.idle":"2022-04-24T16:49:52.745916Z","shell.execute_reply.started":"2022-04-24T16:49:52.729842Z","shell.execute_reply":"2022-04-24T16:49:52.743379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyEarlyStopCallback(EarlyStopCallback):\n    def __init__(\n        self,\n        patience=5,\n        metric=np.less,\n        min_delta=0.0,\n        max_seconds=math.inf,\n        last_k_epochs=3,\n        **kwargs\n    ):\n        super().__init__(**kwargs)\n        self.patience = patience\n        self.min_delta = min_delta\n        self.metric = metric\n        self.max_seconds = max_seconds\n        self.last_k_epochs = last_k_epochs\n        self.bad_epochs = 0\n\n    # diff two gradient of mean\n    def diff(self, losses):\n        k = self.last_k_epochs\n        if k + 1 > len(losses): return False\n        before, after = losses[-k-1:-1], losses[-k:]\n        before, after = np.diff(before).mean(), np.diff(after).mean()\n        # return self.metric(before, after + self.min_delta)\n        return self.metric(after, self.min_delta)\n    \n    def on_train_end(self, epoch: int, time_elapsed: int, info: dict):\n        super().on_train_end(epoch, time_elapsed, info)\n        if time_elapsed >= self.max_seconds: return True\n        return False\n        if self.diff(info['train_loss']) or self.diff(info['valid_loss']):\n            self.bad_epochs += 1\n        else:\n            self.bad_epochs = 0\n        self.message(f'remain {self.patience - self.bad_epochs} patience(s)')\n        return self.bad_epochs >= self.patience","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:49:53.022549Z","iopub.execute_input":"2022-04-24T16:49:53.023229Z","iopub.status.idle":"2022-04-24T16:49:53.033376Z","shell.execute_reply.started":"2022-04-24T16:49:53.023189Z","shell.execute_reply":"2022-04-24T16:49:53.03255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## K-Fold Loader","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import SubsetRandomSampler\n\nclass KFoldDataLoader:\n    def __init__(self, dataset, batch_size=16, **kwargs):\n        self.dataset = dataset\n        self.kwargs = kwargs\n        self.batch_size = batch_size\n        self.reset(**kwargs)\n    \n    def reset(self, **kwargs):\n        self.generator = StratifiedKFold(**kwargs)\n        self.splits = list(self.generator.split(self.dataset.data, self.dataset.targets))\n        self.current = 0\n    \n    def fetch(self):\n        if self.current >= self.generator.get_n_splits():\n            self.reset(**self.kwargs)\n        train_idx, valid_idx = self.splits[self.current]\n        ##### FOR DEBUG\n        rate = 0.75\n        t_count = int(len(train_idx) * rate)\n        v_count = int(len(valid_idx) * rate)\n        train_idx = train_idx[::len(train_idx)//t_count]\n        valid_idx = valid_idx[::len(valid_idx)//v_count]\n        print('Image count: {} trains, {} valids'.format(len(train_idx), len(valid_idx)))\n        ##### FOR DEBUG\n        self.current += 1\n        train_subsampler = SubsetRandomSampler(train_idx)\n        valid_subsampler = SubsetRandomSampler(valid_idx)\n        train_loader = DataLoader(self.dataset, batch_size=self.batch_size, sampler=train_subsampler)\n        valid_loader = DataLoader(self.dataset, batch_size=self.batch_size, sampler=valid_subsampler)\n        return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:02.898161Z","iopub.execute_input":"2022-04-24T16:53:02.898438Z","iopub.status.idle":"2022-04-24T16:53:02.91054Z","shell.execute_reply.started":"2022-04-24T16:53:02.898387Z","shell.execute_reply":"2022-04-24T16:53:02.909833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Dataset and Fold to train","metadata":{}},{"cell_type":"code","source":"dataset = SorghumDataset(df_train, is_train=True, transform=transform_train)\ndataloader_kfold = KFoldDataLoader(dataset, batch_size=Hypers.batch_size, n_splits=4, shuffle=True)\n# train_loader, valid_loader = dataloader_kfold.fetch()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:03.395824Z","iopub.execute_input":"2022-04-24T16:53:03.396349Z","iopub.status.idle":"2022-04-24T16:53:03.41786Z","shell.execute_reply.started":"2022-04-24T16:53:03.39631Z","shell.execute_reply":"2022-04-24T16:53:03.417148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## how long does it take to load all batches","metadata":{}},{"cell_type":"code","source":"# %%time\n# for title, loader in [('train', train_loader), ('valid', valid_loader)]:\n#     labels = []\n#     for batch in loader:\n#         images, cultivars, _ = batch\n#         labels.extend(cultivars)\n#     df1 = pd.DataFrame(labels, columns=['cultivars'])\n#     df2 = df1['cultivars'].value_counts().sort_index()\n#     print('Count of classes:', len(df2))\n#     print('Skew: ', df2.skew())\n\n#     plt.figure(figsize=(20, 4))\n#     plt.title(f'Distribution by {title} loader')\n#     plt.xticks(rotation=90)\n#     sns.barplot(x=df2.index, y=df2.values)\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:03.705878Z","iopub.execute_input":"2022-04-24T16:53:03.70643Z","iopub.status.idle":"2022-04-24T16:53:03.711388Z","shell.execute_reply.started":"2022-04-24T16:53:03.706373Z","shell.execute_reply":"2022-04-24T16:53:03.71047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Clear CUDA memory\n# torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:04.010358Z","iopub.execute_input":"2022-04-24T16:53:04.010705Z","iopub.status.idle":"2022-04-24T16:53:04.014805Z","shell.execute_reply.started":"2022-04-24T16:53:04.010668Z","shell.execute_reply":"2022-04-24T16:53:04.014108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torch.utils.data import random_split\n\n# model = Classifier().to(Hypers.device)\n# print(model)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:04.31808Z","iopub.execute_input":"2022-04-24T16:53:04.318344Z","iopub.status.idle":"2022-04-24T16:53:04.321915Z","shell.execute_reply.started":"2022-04-24T16:53:04.318312Z","shell.execute_reply":"2022-04-24T16:53:04.321129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DenseNet 161","metadata":{}},{"cell_type":"code","source":"densenet161 = torch.hub.load('pytorch/vision:v0.10.0', 'densenet161', pretrained=True)\n\n# Freeze our feature parameters\nfor param in densenet161.parameters():\n    param.requires_grad = False\n\nmodel = densenet161.to(Hypers.device)\nmodel.classifier = nn.Linear(2208, 100)\n\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:04.481824Z","iopub.execute_input":"2022-04-24T16:53:04.482501Z","iopub.status.idle":"2022-04-24T16:53:05.198822Z","shell.execute_reply.started":"2022-04-24T16:53:04.482467Z","shell.execute_reply":"2022-04-24T16:53:05.198097Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EfficientNet","metadata":{}},{"cell_type":"code","source":"# efficientnet = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_efficientnet_b0', pretrained=True)\n\n# # Freeze our feature parameters\n# for param in efficientnet.parameters():\n#     param.requires_grad = False\n\n# model = efficientnet.to(Hypers.device)\n# model.classifier.fc = nn.Linear(1280, 100)\n\n# print(model)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:53:05.200485Z","iopub.execute_input":"2022-04-24T16:53:05.200901Z","iopub.status.idle":"2022-04-24T16:53:05.204936Z","shell.execute_reply.started":"2022-04-24T16:53:05.200863Z","shell.execute_reply":"2022-04-24T16:53:05.204183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Parameters to learn:')\nparameters_learn = []\nfor name, param in model.named_parameters():\n    if param.requires_grad == True:\n        parameters_learn.append(param)\n        print('    ',name)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:56:19.423671Z","iopub.execute_input":"2022-04-24T16:56:19.424741Z","iopub.status.idle":"2022-04-24T16:56:19.434691Z","shell.execute_reply.started":"2022-04-24T16:56:19.424693Z","shell.execute_reply":"2022-04-24T16:56:19.433761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parser_train(batch):\n    image, cultivar, label = batch\n    return image, label\n\n\nargs = {\n    'accelerator': Hypers.device,\n    'loss_function': nn.CrossEntropyLoss(),\n    'optimizer': Hypers.optimizer(parameters_learn, lr=Hypers.learning_rate),\n    'min_epochs': 2,\n    'max_epochs': Hypers.max_epoches,\n    'progress': True,\n    'batch_parser': parser_train,\n    'verbose': Hypers.verbose,\n    'callbacks': [\n        MyEarlyStopCallback(\n            patience=Hypers.patience,\n            min_delta=-1, # gradient to be added\n            max_seconds=5 * 3600,\n            verbose=True),\n#         CheckpointCallback(\n#             name='sorghum_model.pth',\n#             min_delta=0.05,\n#             load_from_checkpoint=True,\n#             verbose=True),\n    ]\n}\n\nprint(args)\n\ntrainer = Trainer(**args)\ntrainer.fit(model, dataloader_kfold.fetch)","metadata":{"execution":{"iopub.status.busy":"2022-04-24T17:10:00.775792Z","iopub.execute_input":"2022-04-24T17:10:00.776416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"logging_df = pd.DataFrame(trainer.log)\n\nplt.figure(figsize=(10, 8))\nsns.lineplot(data=logging_df[['train_loss', 'valid_loss']])\nplt.show()\n\nplt.figure(figsize=(10, 8))\nsns.lineplot(data=logging_df[['train_acc', 'valid_acc']])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:56:16.680643Z","iopub.status.idle":"2022-04-24T16:56:16.681195Z","shell.execute_reply.started":"2022-04-24T16:56:16.680952Z","shell.execute_reply":"2022-04-24T16:56:16.680978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"df_submit = pd.read_csv('../input/sorghum-id-fgvc-9/sample_submission.csv')\ndf_submit.head(5)\n\ntransform_test = T.Compose([\n#     CLAHE,\n    ImageConv.np2tensor,\n    T.Resize(Hypers.image_size),\n    normalize,\n])\n\ntest_dataset = SorghumDataset(df_submit, is_train=False, transform=transform_test)\ntest_loader = DataLoader(test_dataset, batch_size=Hypers.batch_size, shuffle=False)\n\nimages, _, _ = next(iter(test_loader))\nimage_gridplot(images[:12], transform=T.Compose([\n    unnormalize,\n    ImageConv.to_plt,\n]))","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:56:16.682347Z","iopub.status.idle":"2022-04-24T16:56:16.682909Z","shell.execute_reply.started":"2022-04-24T16:56:16.682676Z","shell.execute_reply":"2022-04-24T16:56:16.682703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parser_test(batch):\n    image, label, filename = batch\n    return image, label, filename\n\n# TODO: implement\n# trainer.test(model, test_loader)\n\nresult = {\n    'filename': [],\n    'cultivar': [],\n}\nfor batch in tqdm_nb(test_loader, desc='Get predictions'):\n    images, l, filenames = parser_test(batch)\n    outputs = model(images.to(Hypers.device)).detach().cpu()\n    _, preds = torch.max(outputs.data, 1)\n    preds_decode = dataset.encoder.inverse_transform(preds)\n    result['filename'].extend(filenames)\n    result['cultivar'].extend(preds_decode)\n\nresult_df = pd.DataFrame(result)\nresult_df.to_csv('submission.csv', index=False)\nresult_df","metadata":{"execution":{"iopub.status.busy":"2022-04-24T16:56:16.683988Z","iopub.status.idle":"2022-04-24T16:56:16.684533Z","shell.execute_reply.started":"2022-04-24T16:56:16.684288Z","shell.execute_reply":"2022-04-24T16:56:16.684315Z"},"trusted":true},"execution_count":null,"outputs":[]}]}