{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Installations/Declarations"},{"metadata":{"trusted":true},"cell_type":"code","source":"! cd /kaggle/working","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" #!conda install -y gdown\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### !gdown https://drive.google.com/uc?id=1-26BqndR97eDRxhWGBKObLbTEqR7r3iM\n# !gdown https://drive.google.com/uc?id=1--2oQMhx4IT6hiUvgzYgyKvKINcmnmZe"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport datetime\nimport copy\nimport pandas as pd\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom tqdm import tqdm as tq\nimport psutil\nfrom pathlib import Path\nimport torch\nimport torch.nn as nn \nimport torchvision\nfrom PIL import Image\nfrom sklearn import metrics,model_selection\nimport torchvision.transforms as transforms\nimport albumentations\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!cp '/kaggle/input/resnet50/resnet50.pth' '/root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls /kaggle/input/efficientnet-pytorch/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp '/kaggle/input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth' '/root/.cache/torch/hub/checkpoints/efficientnet-b4-6ed6700e.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cd /kaggle/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH= Path('/kaggle/input/cassava-leaf-disease-classification/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pre-processing / EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(BASE_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"map_classes=pd.read_json(BASE_PATH/'label_num_to_disease_map.json',typ='series')\nmap_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_paths= os.listdir(BASE_PATH/'train_images')\nprint(f\"Number of train images: {len(image_paths)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_shapes = {}\nfor image_name in image_paths[:300]:\n    image = cv2.imread(os.path.join(BASE_PATH/\"train_images\"/image_name))\n    image_shapes[image.shape] = image_shapes.get(image.shape, 0) + 1\nprint(image_shapes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(BASE_PATH/\"train.csv\")\ndf_train[\"class_name\"] = df_train[\"label\"].map(map_classes)\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.title('Distribution of classes')\nsns.countplot(y=\"class_name\", data=df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_images(image_ids, labels):\n    plt.figure(figsize=(20, 20))\n    for ind, (image_id, label) in enumerate(zip(image_ids, labels)):\n        plt.subplot(3, 3, ind + 1)\n        image = cv2.imread(os.path.join(BASE_PATH/\"train_images\"/image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        plt.imshow(image)\n        plt.title(f\"Class: {label},name: {image_id}\", fontsize=12)\n        plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp_df = df_train.sample(9)\nimage_ids = tmp_df[\"image_id\"].values\nlabels = tmp_df[\"class_name\"].values\nview_images(image_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset(torch.utils.data.Dataset):\n    def __init__(self,image_paths, targets,augmentations=None):\n      self.image_paths = image_paths\n      self.targets = targets\n      self.augmentations = augmentations\n    def __len__(self):\n      return len(self.image_paths)\n\n    def __getitem__(self, index):\n      targets= self.targets[index]\n      image = Image.open(self.image_paths[index])\n      image = np.array(image)\n      if self.augmentations is not None:\n        augmented = self.augmentations(image=image)\n        image = augmented[\"image\"]\n        \n      image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n      return{\n          \"image\": torch.tensor(image),\n          \"targets\": torch.tensor(targets)\n          }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_aug=albumentations.Compose(\n        [\n          albumentations.RandomSizedCrop(min_max_height=(224,224),height=224, width=224, p=0.5),\n          albumentations.RandomRotate90(p=0.5),\n          albumentations.VerticalFlip(p=0.5),\n          albumentations.HorizontalFlip(p=0.5),\n          albumentations.Cutout(p=0.5),\n          albumentations.RandomBrightnessContrast(\n              brightness_limit=(-0.1,0.1), \n              contrast_limit=(-0.1, 0.1), \n              p=0.5\n              ),\n          albumentations.Normalize(\n              mean=[0.485, 0.456, 0.406], \n              std=[0.229, 0.224, 0.225], \n              max_pixel_value=255.0, \n              p=1.0\n              ),\n         \n          albumentations.Resize(height=224, width=224, p=1.0)\n        \n        ],\n        p=1.0,     \n    )\n\nvalid_aug = albumentations.Compose(\n        [\n          albumentations.Resize(height=224, width=224, p=1.0),\n          albumentations.RandomBrightnessContrast(\n              brightness_limit=(-0.1,0.1), \n              contrast_limit=(-0.1, 0.1), \n              p=0.5\n              ),\n          albumentations.Normalize(\n              mean=[0.485, 0.456, 0.406], \n              std=[0.229, 0.224, 0.225], \n              max_pixel_value=255.0, \n              p=1.0\n              )\n        ],\n        p=1.0,\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train/Validation Datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset=ImageDataset(\n#     image_paths=train_image_paths,\n#     targets=train_targets,\n#     augmentations=train_aug\n# )\n# valid_dataset=ImageDataset(\n#     image_paths=valid_image_paths,\n#     targets=valid_targets,\n#     augmentations=valid_aug\n# )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code taken from https://github.com/fhopfmueller/bi-tempered-loss-pytorch/blob/master/bi_tempered_loss_pytorch.py\n\ndef log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://github.com/abhishekkrthakur/tez/blob/main/tez/model/model.py\nclass Model(nn.Module):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.train_loader = None\n        self.valid_loader = None\n        self.optimizer = None\n        self.scheduler = None\n        self.step_scheduler_after = None\n        self.step_scheduler_metric = None\n        self.current_epoch = 0\n        self.current_train_step = 0\n        self.current_valid_step = 0\n        self._model_state = None\n        self._train_state = None\n        self._callback_runner = None\n        self.fp16 = False\n        self.scaler = None\n        self.metrics = {}\n        self.metrics[\"train\"] = {}\n        self.metrics[\"valid\"] = {}\n        self.metrics[\"test\"] = {}\n\n    @property\n    def model_state(self):\n        return self._model_state\n\n    @model_state.setter\n    def model_state(self, value):\n        self._model_state = value\n        # run something here in future if needed\n\n    @property\n    def train_state(self):\n        return self._train_state\n\n    @train_state.setter\n    def train_state(self, value):\n        self._train_state = value\n        if self._callback_runner is not None:\n            self._callback_runner(value)\n\n    def _init_model(\n        self,\n        device,\n        train_dataset,\n        valid_dataset,\n        train_sampler,\n        valid_sampler,\n        train_bs,\n        valid_bs,\n        n_jobs,\n        callbacks,\n        fp16,\n    ):\n\n        if callbacks is None:\n            callbacks = list()\n\n        if n_jobs == -1:\n            n_jobs = psutil.cpu_count()\n\n        if next(self.parameters()).device != device:\n            self.to(device)\n\n        if self.train_loader is None:\n            self.train_loader = torch.utils.data.DataLoader(\n                train_dataset,\n                batch_size=train_bs,\n                num_workers=n_jobs,\n                sampler=train_sampler,\n                shuffle=True,\n            )\n        if self.valid_loader is None:\n            if valid_dataset is not None:\n                self.valid_loader = torch.utils.data.DataLoader(\n                    valid_dataset,\n                    batch_size=valid_bs,\n                    num_workers=n_jobs,\n                    sampler=valid_sampler,\n                    shuffle=False,\n                )\n\n        if self.optimizer is None:\n            self.optimizer = self.fetch_optimizer()\n\n        if self.scheduler is None:\n            self.scheduler = self.fetch_scheduler()\n\n        self.fp16 = fp16\n        if self.fp16:\n            self.scaler = torch.cuda.amp.GradScaler()\n\n        self._callback_runner = CallbackRunner(callbacks, self)\n        self.train_state = TrainingState.TRAIN_START\n\n    def monitor_metrics(self, *args, **kwargs):\n        return\n\n    def loss(self, *args, **kwargs):\n        return\n\n    def fetch_optimizer(self, *args, **kwargs):\n        return\n\n    def fetch_scheduler(self, *args, **kwargs):\n        return\n\n    def forward(self, *args, **kwargs):\n        return super().forward(*args, **kwargs)\n\n    def train_one_step(self, data, device):\n        self.optimizer.zero_grad()\n        for key, value in data.items():\n            data[key] = value.to(device)\n        if self.fp16:\n            with torch.cuda.amp.autocast():\n                _, loss, metrics = self(**data)\n        else:\n            _, loss, metrics = self(**data)\n        with torch.set_grad_enabled(True):\n            if self.fp16:\n                with torch.cuda.amp.autocast():\n                    self.scaler.scale(loss).backward()\n                    self.scaler.step(self.optimizer)\n                    self.scaler.update()\n            else:\n                loss.backward()\n                self.optimizer.step()\n            if self.scheduler:\n                if self.step_scheduler_after == \"batch\":\n                    if self.step_scheduler_metric is None:\n                        self.scheduler.step()\n                    else:\n                        self.scheduler.step(self.step_scheduler_metric)\n        return loss, metrics\n\n    def validate_one_step(self, data, device):\n        for key, value in data.items():\n            data[key] = value.to(device)\n        _, loss, metrics = self(**data)\n        return loss, metrics\n\n    def predict_one_step(self, data, device):\n        for key, value in data.items():\n            data[key] = value.to(device)\n        output, _, _ = self(**data)\n        return output\n\n    def update_metrics(self, losses, monitor):\n        self.metrics[self._model_state.value].update(monitor)\n        self.metrics[self._model_state.value][\"loss\"] = losses.avg\n\n    def train_one_epoch(self, data_loader, device):\n        self.train()\n        self.model_state = ModelState.TRAIN\n        losses = AverageMeter()\n        tk0 = tq(data_loader, total=len(data_loader))\n        for b_idx, data in enumerate(tk0):\n            self.train_state =TrainingState.TRAIN_STEP_START\n            loss, metrics = self.train_one_step(data, device)\n            self.train_state = TrainingState.TRAIN_STEP_END\n            losses.update(loss.item(), data_loader.batch_size)\n            if b_idx == 0:\n                metrics_meter = {k: AverageMeter() for k in metrics}\n            monitor = {}\n            for m_m in metrics_meter:\n                metrics_meter[m_m].update(metrics[m_m], data_loader.batch_size)\n                monitor[m_m] = metrics_meter[m_m].avg\n            self.current_train_step += 1\n            tk0.set_postfix(loss=losses.avg, stage=\"train\", **monitor)\n        tk0.close()\n        self.update_metrics(losses=losses, monitor=monitor)\n        return losses.avg\n\n    def validate_one_epoch(self, data_loader, device):\n        self.eval()\n        self.model_state = ModelState.VALID\n        losses = AverageMeter()\n        tk0 = tq(data_loader, total=len(data_loader))\n        for b_idx, data in enumerate(tk0):\n            self.train_state = TrainingState.VALID_STEP_START\n            with torch.no_grad():\n                loss, metrics = self.validate_one_step(data, device)\n            self.train_state = TrainingState.VALID_STEP_END\n            losses.update(loss.item(), data_loader.batch_size)\n            if b_idx == 0:\n                metrics_meter = {k: AverageMeter() for k in metrics}\n            monitor = {}\n            for m_m in metrics_meter:\n                metrics_meter[m_m].update(metrics[m_m], data_loader.batch_size)\n                monitor[m_m] = metrics_meter[m_m].avg\n            tk0.set_postfix(loss=losses.avg, stage=\"valid\", **monitor)\n            self.current_valid_step += 1\n        tk0.close()\n        self.update_metrics(losses=losses, monitor=monitor)\n        return losses.avg\n\n    def process_output(self, output):\n        output = output.cpu().detach().numpy()\n        return output\n\n    def predict(self, dataset, device, sampler=None, batch_size=16, n_jobs=1):\n        if next(self.parameters()).device != device:\n            self.to(device)\n\n        if n_jobs == -1:\n            n_jobs = psutil.cpu_count()\n\n        data_loader = torch.utils.data.DataLoader(\n            dataset, batch_size=batch_size, num_workers=n_jobs, sampler=sampler\n        )\n        self.eval()\n        final_output = []\n        tk0 = tq(data_loader, total=len(data_loader))\n        for b_idx, data in enumerate(tk0):\n            with torch.no_grad():\n                out = self.predict_one_step(data, device)\n                out = self.process_output(out)\n                yield out\n            tk0.set_postfix(stage=\"test\")\n        tk0.close()\n\n    def save(self, model_path):\n        model_state_dict = self.state_dict()\n        if self.optimizer is not None:\n            opt_state_dict = self.optimizer.state_dict()\n        else:\n            opt_state_dict = None\n        if self.scheduler is not None:\n            sch_state_dict = self.scheduler.state_dict()\n        else:\n            sch_state_dict = None\n        model_dict = {}\n        model_dict[\"state_dict\"] = model_state_dict\n        model_dict[\"optimizer\"] = opt_state_dict\n        model_dict[\"scheduler\"] = sch_state_dict\n        model_dict[\"epoch\"] = self.current_epoch\n        model_dict[\"fp16\"] = self.fp16\n        torch.save(model_dict, model_path)\n\n    def load(self, model_path, device=\"cuda\"):\n        if next(self.parameters()).device != device:\n            self.to(device)\n        model_dict = torch.load(model_path)\n        self.load_state_dict(model_dict[\"state_dict\"])\n\n    def fit(\n        self,\n        train_dataset,\n        valid_dataset=None,\n        train_sampler=None,\n        valid_sampler=None,\n        device=\"cuda\",\n        epochs=10,\n        train_bs=16,\n        valid_bs=16,\n        n_jobs=8,\n        callbacks=None,\n        fp16=False,\n    ):\n        self._init_model(\n            device=device,\n            train_dataset=train_dataset,\n            valid_dataset=valid_dataset,\n            train_sampler=train_sampler,\n            valid_sampler=valid_sampler,\n            train_bs=train_bs,\n            valid_bs=valid_bs,\n            n_jobs=n_jobs,\n            callbacks=callbacks,\n            fp16=fp16,\n        )\n\n        for _ in range(epochs):\n            self.train_state =TrainingState.EPOCH_START\n            self.train_state =TrainingState.TRAIN_EPOCH_START\n            train_loss = self.train_one_epoch(self.train_loader, device)\n            self.train_state = TrainingState.TRAIN_EPOCH_END\n            if self.valid_loader:\n                self.train_state =TrainingState.VALID_EPOCH_START\n                valid_loss = self.validate_one_epoch(self.valid_loader, device)\n                self.train_state = TrainingState.VALID_EPOCH_END\n            if self.scheduler:\n                if self.step_scheduler_after == \"epoch\":\n                    if self.step_scheduler_metric is None:\n                        self.scheduler.step()\n                    else:\n                        self.scheduler.step(self.step_scheduler_metric)\n            self.train_state =TrainingState.EPOCH_END\n            if self._model_state.value == \"end\":\n                break\n            self.current_epoch += 1\n        self.train_state =TrainingState.TRAIN_END","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* AverageMeter "},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n#Computes and stores the average and current value\n\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Model+Training state specifier "},{"metadata":{"trusted":true},"cell_type":"code","source":"from enum import Enum\nclass ModelState(Enum):\n    TRAIN = \"train\"\n    VALID = \"valid\"\n    TEST = \"test\"\n    END = \"end\"\n\n\nclass TrainingState(Enum):\n    TRAIN_START = \"on_train_start\"\n    TRAIN_END = \"on_train_end\"\n    EPOCH_START = \"on_epoch_start\"\n    EPOCH_END = \"on_epoch_end\"\n    TRAIN_EPOCH_START = \"on_train_epoch_start\"\n    TRAIN_EPOCH_END = \"on_train_epoch_end\"\n    VALID_EPOCH_START = \"on_valid_epoch_start\"\n    VALID_EPOCH_END = \"on_valid_epoch_end\"\n    TRAIN_STEP_START = \"on_train_step_start\"\n    TRAIN_STEP_END = \"on_train_step_end\"\n    VALID_STEP_START = \"on_valid_step_start\"\n    VALID_STEP_END = \"on_valid_step_end\"\n    TEST_STEP_START = \"on_test_step_start\"\n    TEST_STEP_END = \"on_test_step_end\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Callback"},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import List, Tuple\n\n\nclass Callback:\n    def on_epoch_start(self, model, **kwargs):\n        return\n\n    def on_epoch_end(self, model, **kwargs):\n        return\n\n    def on_train_epoch_start(self, model, **kwargs):\n        return\n\n    def on_train_epoch_end(self, model, **kwargs):\n        return\n\n    def on_valid_epoch_start(self, model, **kwargs):\n        return\n\n    def on_valid_epoch_end(self, model, **kwargs):\n        return\n\n    def on_train_step_start(self, model, **kwargs):\n        return\n\n    def on_train_step_end(self, model, **kwargs):\n        return\n\n    def on_valid_step_start(self, model, **kwargs):\n        return\n\n    def on_valid_step_end(self, model, **kwargs):\n        return\n\n    def on_test_step_start(self, model, **kwargs):\n        return\n\n    def on_test_step_end(self, model, **kwargs):\n        return\n\n    def on_train_start(self, model, **kwargs):\n        return\n\n    def on_train_end(self, model, **kwargs):\n        return\n\n\nclass CallbackRunner:\n    def __init__(self, callbacks: List[Callback], model):\n        self.model = model\n        self.callbacks = callbacks\n\n    def __call__(self, current_state, **kwargs):\n        for cb in self.callbacks:\n            _ = getattr(cb, current_state.value)(self.model, **kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EarlyStopping(Callback):\n    def __init__(self, monitor, model_path, patience=5, mode=\"min\", delta=0.001,epochs=None):\n        self.monitor = monitor\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.delta = delta\n        self.model_path = model_path\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n        if self.monitor.startswith(\"train_\"):\n            self.model_state = \"train\"\n            self.monitor_value = self.monitor[len(\"train_\") :]\n        elif self.monitor.startswith(\"valid_\"):\n            self.model_state = \"valid\"\n            self.monitor_value = self.monitor[len(\"valid_\") :]\n        else:\n            raise Exception(\"monitor must start with train_ or valid_\")\n\n    def on_epoch_end(self, model):\n        epoch_score = model.metrics[self.model_state][self.monitor_value]\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(\n                \"EarlyStopping counter: {} out of {}\".format(\n                    self.counter, self.patience\n                )\n            )\n            if self.counter >= self.patience:\n                model.model_state = ModelState.END\n        else:\n            self.best_score = score\n            self.save_checkpoint(epoch_score, model)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            print(\n                \"Validation score improved ({} --> {}). Saving model!\".format(\n                    self.val_score, epoch_score\n                )\n            )\n            model.save(self.model_path)\n        self.val_score = epoch_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{},"cell_type":"markdown","source":"<h2>EfficientNet</h2>"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Cassava_Efficientnet(Model):\n  def __init__(self,num_classes,pretrained=True):\n    super().__init__()\n    self.convnet= EfficientNet.from_pretrained('efficientnet-b4', num_classes=num_classes)\n    self.step_scheduler_after=\"epoch\"\n\n  \n  def loss(self,outputs,targets):\n    if targets is None:\n      return None \n    return nn.CrossEntropyLoss()(outputs,targets)  \n\n  def monitor_metrics(self,outputs,targets):\n    _, preds = torch.max(outputs,1)\n    acc=bi_tempered_logistic_loss(outputs,targets, t1=0.8, t2=1.4, label_smoothing=0.2)\n    return {\n        \"bi_tempered_logistic_loss\":acc\n    }\n\n\n  def fetch_optimizer(self):\n    opt=torch.optim.Adam(self.parameters(),lr=1e-3)\n    return opt\n\n  def fetch_scheduler(self):\n    sch=torch.optim.lr_scheduler.StepLR(self.optimizer,step_size=0.7)\n    return sch\n\n  def forward(self,image,targets=None):\n    outputs=self.convnet(image)\n    if targets is not None:\n      loss=self.loss(outputs,targets)\n      mon_metrics=self.monitor_metrics(outputs,targets)\n      return outputs,loss,mon_metrics  \n    return outputs,None,None  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_Efficientnet=Cassava_Efficientnet(num_classes=df_train.label.nunique(),pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Train Efficientnet"},{"metadata":{"trusted":true},"cell_type":"code","source":" model_Efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train= df_train.sample(frac=1).reset_index(drop=True)\n# initiate the kfold class from model_selection module\nkf = model_selection.StratifiedKFold(n_splits=5)\n# # fill the new kfold column\ncvscores = []\nFold = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#model_name = 'Efficientnet_Fold_'+str(Fold)+'.h5'\n#model_Efficientnet.save(BASE_PATH/'cassava_models'/model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path=BASE_PATH/'train_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" for train,val in kf.split(X=df_train,y=df_train['label']):\n   X_train=df_train.iloc[train,:].reset_index(drop=True)\n   X_val=df_train.iloc[val,:].reset_index(drop=True)\n   y_train=X_train.label.values\n   y_val=X_val.label.values\n   train_image_paths=[\n                    os.path.join(image_path,x) for x in X_train.image_id.values\n   ]\n   valid_image_paths=[\n                    os.path.join(image_path,x) for x in X_val.image_id.values\n   ]\n   train_dataset=ImageDataset(\n     image_paths=train_image_paths,\n     targets=y_train,\n     augmentations=train_aug\n   )\n   valid_dataset=ImageDataset(\n     image_paths=valid_image_paths,\n     targets=y_val,\n     augmentations=valid_aug\n   )\n   train_loader = torch.utils.data.DataLoader(\n                 train_dataset,\n                 batch_size=32,\n                 num_workers=psutil.cpu_count(),    #n_jobs,\n                 sampler=None,\n                 shuffle=True,\n             )\n\n   es=EarlyStopping(\n     monitor=\"valid_bi_tempered_logistic_loss\",\n     model_path=\"/kaggle/working/\"+'Efficientnet_Fold_'+str(Fold)+\".bin\",\n     patience=3,\n     mode=\"min\"\n     )\n   model_Efficientnet.fit(\n     train_dataset,\n     valid_dataset=valid_dataset,\n     train_bs=32,\n     valid_bs=32,\n     device=\"cuda\",\n     callbacks=[es],\n     fp16=True,\n     epochs=50\n   )\n   Fold=Fold+1  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" es=EarlyStopping(\n     monitor=\"valid_accuracy\",\n     model_path=\"/kaggle/working/Effmodel.bin\",\n     patience=3,\n     mode=\"max\"\n     )\n model_Efficientnet.fit(\n     train_dataset,\n     valid_dataset=valid_dataset,\n     train_bs=32,\n     valid_bs=64,\n     device=None,\n     callbacks=[es],\n     fp16=True,\n     epochs=50\n )\n\nmodel_name = 'model_Efficientnet.pt'\ntorch.save(model_Efficientnet.state_dict(), model_name) \nfrom IPython.display import FileLink\nFileLink(r'model_Efficientnet.pt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# For submission "},{"metadata":{"trusted":true},"cell_type":"code","source":"model_Efficientnet.load(\"/kaggle/input/colab11/Efficientnet_Fold_5.bin\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit = pd.read_csv(BASE_PATH/\"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_paths=[\n                   os.path.join(BASE_PATH/\"test_images\",x) for x in df_submit.image_id.values\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_targets=df_submit.label.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = ImageDataset(\n    image_paths=test_image_paths,\n    targets=test_targets,\n    augmentations=valid_aug\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_resnet= Cassava_Resnet(num_classes=df_train.label.nunique())\n# model_resnet.load(\"/kaggle/working/resmodel.bin\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def Emb(model1,model2,test_dataset,batch_size=32):\n#     preds1=model1.predict(test_dataset, batch_size=32, n_jobs=-1, device=\"cuda\")\n#     preds2=model2.predict(test_dataset, batch_size=32, n_jobs=-1, device=\"cuda\")\n#     return ((np.vstack(list(preds1))+np.vstack(list(preds2)))/2).argmax(axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds=model_Efficientnet.predict(test_dataset, batch_size=32, n_jobs=-1, device=\"cuda\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res=np.vstack(list(preds)).argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(0,3):\n    temp_preds=model_Efficientnet.predict(test_dataset, batch_size=32, n_jobs=-1, device=\"cuda\")\n    res=res+np.vstack(list(temp_preds))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res=res/5\nres=res.argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.label=res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.to_csv(\"/kaggle/working/submission.csv\", index=False,header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}