{"cells":[{"metadata":{},"cell_type":"markdown","source":"# https://www.kaggle.com/ihelon/cassava-leaf-disease-exploratory-data-analysis\n\n# https://www.kaggle.com/yasufuminakama/cassava-resnext50-32x4d-starter-training/\n\n# https://www.kaggle.com/abhishek/leaf-disease-inference-using-tez\n\n# https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta"},{"metadata":{},"cell_type":"markdown","source":"# Things to add\n\n* [x] Data augmentation\n* [ ] Offline data augmentation\n* [x] TTA\n* [ ] Scheduler\n* [ ] Dropout before classifier\n* [ ] image size 512\n* [ ] Error analysis\n* [ ] Model predictions as original label (soft-labelling)\n* [ ] Different model architectures\n* [ ] Ensemble models"},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"# !pip install pretrainedmodels\n!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\n\nimport cv2\nimport albumentations\n\nimport timm\n# import pretrainedmodels as pm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nfrom contextlib import contextmanager\n\nLOGS_PATH = Path(\"logs\")\nLOGS_PATH.mkdir(exist_ok=True)\n\n\ndef init_logger(log_file=LOGS_PATH / 'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n\nLOGGER = init_logger()\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":false},"cell_type":"code","source":"label_mapping_fn = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\nwith open(label_mapping_fn, \"r\") as f:\n    labels = json.loads(f.read())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_img_dir = \"../input/cassava-leaf-disease-classification/train_images/\"\ntrain_img_files = list(Path(train_img_dir).iterdir())\nprint(\"Total train images: \", len(train_img_files))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_csv_fn = \"../input/cassava-leaf-disease-classification/train.csv\"\ndf_train = pd.read_csv(train_csv_fn)\n\ndf_train[\"class_name\"] = df_train[\"label\"].astype(str).map(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"df_train.class_name.value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"BASE_IMG_DIR = Path(\"../input/cassava-leaf-disease-classification/train_images/\")\n\ndef read_img_and_cvt_format(img_path, clr_format=cv2.COLOR_BGR2RGB):\n    return cv2.cvtColor(cv2.imread(img_path), clr_format)\n\ndef visualize_batch(img_ids, labels):\n    \n    plt.figure(figsize=(16, 12))\n    \n    for idx, (img_id, label) in enumerate(zip(img_ids, labels)):\n        plt.subplot(3, 3, idx + 1)\n        img_fn = str(BASE_IMG_DIR / img_id)\n        img = read_img_and_cvt_format(img_fn)\n        plt.imshow(img)\n        plt.title(f\"Class: {label}\", fontsize=9)\n        plt.axis(\"off\")\n        \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"sampled_df = df_train.sample(9)\nimg_ids = sampled_df[\"image_id\"].values\nlabels = sampled_df[\"class_name\"].values\n\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"def sample_with_label(df, label, sample_size=9):\n    \n    filtered_df = df[df[\"label\"] == label]\n    sampled_df = filtered_df.sample(sample_size)\n    img_ids = sampled_df[\"image_id\"].values\n    class_names = sampled_df[\"class_name\"].values\n    \n    return img_ids, class_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"current_label = 0\nimg_ids, labels = sample_with_label(df_train, current_label)\n\nprint(f\"Sample images for the class: {labels[current_label]}\")\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"current_label = 1\nimg_ids, labels = sample_with_label(df_train, current_label)\n\nprint(f\"Sample images for the class: {labels[current_label]}\")\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"current_label = 2\nimg_ids, labels = sample_with_label(df_train, current_label)\n\nprint(f\"Sample images for the class: {labels[current_label]}\")\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"current_label = 3\nimg_ids, labels = sample_with_label(df_train, current_label)\n\nprint(f\"Sample images for the class: {labels[current_label]}\")\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"current_label = 4\nimg_ids, labels = sample_with_label(df_train, current_label)\n\nprint(f\"Sample images for the class: {labels[current_label]}\")\nvisualize_batch(img_ids, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"ssr_aug = albumentations.ShiftScaleRotate(\n    shift_limit=(-0.1, 0.1),\n    scale_limit=(-0.1, 0.1),\n    rotate_limit=(-180, 180),\n    interpolation=0,\n    border_mode=4,\n    p=1.0\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"random_row = df_train.sample(1, random_state=42).values[0]\nrandom_img_id, random_label, random_class_name = random_row\nsingle_img_path = f\"../input/cassava-leaf-disease-classification/train_images/{random_img_id}\"\nrandom_img = read_img_and_cvt_format(single_img_path)\n\nplt.title(f\"Class: {random_class_name}\")\nplt.imshow(random_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"augmented_random_img = ssr_aug(image=random_img)[\"image\"]\n\nplt.imshow(augmented_random_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\n\nclass CassavaDataset(Dataset):\n    \n    def __init__(self, image_paths, labels=None, transform=None):\n        \n        self.image_paths = image_paths\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.image_paths)\n    \n    def __getitem__(self, idx):\n        \n        img_filepath = self.image_paths[idx]\n        img = read_img_and_cvt_format(img_filepath)\n        if self.transform:\n            img = self.transform(image=img)[\"image\"]\n        \n        label = 0\n        if self.labels is not None:\n            label = torch.tensor(self.labels[idx]).long()\n        return img, label\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_IMG_DIR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in df_train[\"image_id\"].values]\ntrain_dataset = CassavaDataset(image_paths=train_img_paths, \n                               labels=df_train[\"label\"].values,\n                               transform=None)\n\nfor i in range(1):\n    img, label = train_dataset[i]\n    \n    plt.title(f\"Label: {label}\")\n    plt.imshow(img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    \n    model_name = \"seresnext50_32x4d\"\n    n_epochs = 10\n    batch_size = 16\n    img_size = 512\n    n_classes = 5\n    lr = 1e-4\n    weight_decay = 1e-6\n    gradient_accumulation_steps = 2\n    max_grad_norm = 1000\n    seed = 42\n    scheduler = \"\"\n    n_fold = 5\n    train_fold = [0, 1, 2, 3, 4]\n    train = True\n    print_every = 100\n    num_workers = 4\n    tta_steps = 1\n    \n\n    \nimport os\nimport random\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=Config.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nimport torch.nn as nn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cross Validation split.\n\ndef create_cv_split(df, n_splits, *args, **kwargs):\n    \n    fold = StratifiedKFold(n_splits=n_splits, *args, **kwargs)\n    for idx, (train_idx, val_idx) in enumerate(fold.split(df, df[\"label\"])):\n        df.loc[val_idx, \"fold\"] = idx\n        \n    df[\"fold\"] = df[\"fold\"].astype(int)\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_folds = create_cv_split(df_train.copy(), \n#                 n_splits=Config.n_fold, \n#                 shuffle=True, \n#                 random_state=Config.seed)\n\n# df_folds.groupby([\"fold\", \"label\"]).size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    \n    def __init__(self, model_name, pretrained=False):\n        super(Classifier, self).__init__()\n        \n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, Config.n_classes)\n        \n    def forward(self, x):\n        return self.model(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations.pytorch import ToTensorV2\nfrom torchvision import transforms as T\n\n# Random augmentation for the train images from the 3rd place solution \n# in the previous competition.\n# def train_transform(size):\n#     return T.Compose([\n#         T.RandomApply([T.RandomAffine(45, shear=15)], 0.8),\n#         RandomResizedCropV2(size, scale=(0.6, 1.0), ratio=(3/5, 5/3)),\n#         T.RandomHorizontalFlip(),\n#         T.RandomVerticalFlip(),\n#         T.ToTensor(),\n#         T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n#         RandomErasing(probability=0.3, sh=0.3),\n#     ])\n\n#     return T.Compose([\n#             T.RandomApply([T.RandomAffine(45, shear=15)], 0.8),\n#             T.RandomResizedCrop(Config.img_size, \n#                                 scale=(0.6, 1.0), \n#                                 ratio=(3/5, 5/3)),\n#             T.RandomHorizontalFlip(),\n#             T.RandomVerticalFlip(),\n#             T.Normalize(mean=[0.485, 0.456, 0.406], \n#                         std=[0.229, 0.224, 0.225]),\n#             T.ToTensor(),\n#             T.RandomErasing(p=0.3)\n#         ])\n\ndef _get_train_transforms_without_aug():\n    return albumentations.Compose([\n#         albumentations.RandomResizedCrop(\n#             Config.img_size, \n#             Config.img_size\n#         ),\n        albumentations.CenterCrop(\n            Config.img_size,\n            Config.img_size\n        ),\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406], \n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n    ])\n\n\ndef get_train_transforms():\n    return albumentations.Compose([\n#         albumentations.RandomResizedCrop(Config.img_size, \n#                                          Config.img_size,\n#                                          scale=(0.6, 1.0), \n#                                          ratio=(3/5, 5/3)),\n        albumentations.CenterCrop(\n            Config.img_size,\n            Config.img_size\n        ),\n        albumentations.Transpose(),\n        albumentations.HorizontalFlip(),\n        albumentations.VerticalFlip(),\n        albumentations.ShiftScaleRotate(),\n        albumentations.HueSaturationValue(\n            hue_shift_limit=0.2, \n            sat_shift_limit=0.2, \n            val_shift_limit=0.2\n        ),\n        albumentations.RandomBrightnessContrast(\n            brightness_limit=(-0.15, 0.15), \n            contrast_limit=(-0.15, 0.15)\n        ),\n        albumentations.Normalize(\n            mean=[0.485, 0.456, 0.406], \n            std=[0.229, 0.224, 0.225]),\n#         albumentations.CoarseDropout(p=0.3),\n        ToTensorV2()\n    ])\n\n\ndef get_test_transforms():\n    \n    return albumentations.Compose([\n        albumentations.Resize(Config.img_size, Config.img_size),\n        albumentations.Normalize(mean=[0.485, 0.456, 0.406], \n                  std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n#     return albumentations.Compose([\n#         # albumentations.Resize(Config.img_size, Config.img_size),\n#         albumentations.RandomResizedCrop(Config.img_size, Config.img_size),\n#         albumentations.Transpose(p=0.5),\n#         albumentations.HorizontalFlip(p=0.5),\n#         albumentations.VerticalFlip(p=0.5),\n#         albumentations.HueSaturationValue(\n#             hue_shift_limit=0.2,\n#             sat_shift_limit=0.2,\n#             val_shift_limit=0.2,\n#             p=0.5\n#         ),\n#         albumentations.RandomBrightnessContrast(\n#             brightness_limit=(-0.1, 0.1),\n#             contrast_limit=(-0.1, 0.1),\n#             p=0.5\n#         ),\n#         albumentations.Normalize(mean=[0.485, 0.456, 0.406], \n#                                  std=[0.229, 0.224, 0.225]),\n#         ToTensorV2()\n#     ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_transforms = get_train_transforms()\n\n# train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in df_train[\"image_id\"].values]\n\n# model = Classifier(Config.model_name, pretrained=False)\n# train_dataset = CassavaDataset(image_paths=train_img_paths, \n#                                labels=df_train[\"label\"].values,\n#                                transform=train_transforms)\n\n# train_data_loader = DataLoader(train_dataset, batch_size=4, \n#                                shuffle=True, num_workers=4)\n\n# for img, label in train_data_loader:\n    \n#     output = model(img)\n#     print(output)\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for img, label in train_data_loader:\n    \n#     output = model(img)\n#     print(output)\n#     break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utilities"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\n\nclass AverageMeter:\n    \n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n    \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \n        \ndef as_minutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return f\"{m}m {s}s\"\n\n\ndef time_since(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / percent\n    rs = es - s\n    return f\"{as_minutes(s)} (remain {as_minutes(rs)})\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_step(model, data_loader, criterion, optimizer, epoch, scheduler, device):\n    \"\"\"\n    There is no scheduler update currently.\n    \"\"\"\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    # scores = AverageMeter()\n    \n    model.train()\n    start = end = time.time()\n    # global_step = 0\n    total_len = len(data_loader)\n    \n    for step, (images, labels) in enumerate(data_loader):\n        \n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        preds = model(images)\n        loss = criterion(preds, labels)\n        losses.update(loss.item(), batch_size)\n        \n        if Config.gradient_accumulation_steps > 1:\n            loss = loss / Config.gradient_accumulation_steps\n        \n        loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), \n                                                   Config.max_grad_norm)\n        if (step + 1) % Config.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            # global_step += 1\n        \n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % Config.print_every == 0 or step == (total_len - 1):\n            print(f\"Epoch: [{epoch+1}][{step}/{total_len}] \"\n                  f\"Data: {data_time.val:.3f} ({data_time.avg:.3f}) \"\n                  f\"Batch: {batch_time.val:.3f} ({batch_time.avg:.3f}) \"\n                  f\"Elapsed: {time_since(start, float(step + 1) / (total_len))} \"\n                  f\"Loss: {losses.val:.5f}({losses.avg:.5f}) \"\n                  f\"Grad: {grad_norm:.4f}\" # LR: {lr:.6f}\n                 )\n    \n    return losses.avg\n            \n\ndef valid_step(model, data_loader, criterion, device):\n    \n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    \n    model.eval()\n    start = end = time.time()\n    total_len = len(data_loader)\n    predictions = []\n    \n    for step, (images, labels) in enumerate(data_loader):\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        \n        with torch.no_grad():\n            preds = model(images)\n        \n        loss = criterion(preds, labels)\n        losses.update(loss.item(), batch_size)\n        predictions.append(preds.softmax(1).cpu().numpy())\n        \n        if Config.gradient_accumulation_steps > 1:\n            loss = loss / Config.gradient_accumulation_steps\n            \n        batch_time.update(time.time() - end)\n        end = time.time()\n        \n        if step % Config.print_every == 0 or step == (total_len - 1):\n            print(f\"Eval: [{step}/{total_len}] \"\n                  f\"Data: {data_time.val:.3f} ({data_time.avg:.3f}) \"\n                  f\"Batch: {batch_time.val:.3f} ({batch_time.avg:.3f}) \"\n                  f\"Elapsed: {time_since(start, float(step + 1) / total_len)} \"\n                  f\"Loss: {losses.val:.5f} ({losses.avg:.5f})\"\n                 )\n    \n    predictions = np.concatenate(predictions)\n    return losses.avg, predictions\n\n\ndef inference(model, states, data_loader, device):\n    \n    model.to(device)\n    tk0 = tqdm(enumerate(data_loader), total=len(data_loader))\n    probs = []\n    \n    for idx, (images, _) in tk0:\n        \n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state[\"model\"])\n            model.eval()\n            with torch.no_grad():\n                preds = model(images)\n            \n            avg_preds.append(preds.softmax(1).cpu().numpy())\n        \n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    \n    probs = np.concatenate(probs)\n    return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\nfrom sklearn.metrics import accuracy_score, classification_report\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nMODELS_DIR = Path(\"models\")\nMODELS_DIR.mkdir(exist_ok=False)\n\n\ndef train_loop(folds, fold):\n    \n    LOGGER.info(f\"========= fold: {fold} training ===========\")\n    train_indices = folds[folds[\"fold\"] != fold].index\n    valid_indices = folds[folds[\"fold\"] == fold].index\n    \n    train_folds = folds.loc[train_indices].reset_index(drop=True)\n    valid_folds = folds.loc[valid_indices].reset_index(drop=True)\n\n    train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in train_folds[\"image_id\"].values]\n    valid_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in valid_folds[\"image_id\"].values]\n    \n    train_dataset = CassavaDataset(\n        train_img_paths, \n        labels=train_folds[\"label\"].values, \n        transform=get_train_transforms()\n    )\n    \n    valid_dataset = CassavaDataset(\n        valid_img_paths,\n        labels=valid_folds[\"label\"].values,\n        transform=get_test_transforms()\n    )\n    \n    train_data_loader = DataLoader(\n        train_dataset, batch_size=Config.batch_size, \n        shuffle=True, num_workers=Config.num_workers\n    )\n    valid_data_loader = DataLoader(\n        valid_dataset, batch_size=Config.batch_size, \n        shuffle=False, num_workers=Config.num_workers\n    )\n    \n    model = Classifier(Config.model_name, pretrained=True)\n    model.to(device)\n    # amsgrad = False\n    optimizer = optim.Adam(model.parameters(), \n                           lr=Config.lr, \n                           weight_decay=Config.weight_decay)\n    criterion = nn.CrossEntropyLoss()\n    best_score = 0.0\n    best_loss = np.inf\n\n    for epoch in range(Config.n_epochs):\n        \n        start_time = time.time()\n        avg_epoch_loss = train_step(model, \n                                    train_data_loader, \n                                    criterion, \n                                    optimizer, \n                                    epoch, \n                                    scheduler=None, \n                                    device=device)\n\n        avg_valid_loss, valid_preds = valid_step(model, \n                                                 valid_data_loader, \n                                                 criterion, \n                                                 device)\n        valid_labels = valid_folds[\"label\"].values\n        accuracy = accuracy_score(valid_labels, valid_preds.argmax(1))\n        classification_result = classification_report(valid_labels, \n                                                      valid_preds.argmax(1))\n        elapsed = time.time() - start_time\n        LOGGER.info(f\"Epoch: {epoch+1} - avg_epoch_loss: {avg_epoch_loss:.5f} - avg_val_loss: {avg_valid_loss:.5f} - time: {elapsed:.0f}s\")\n        LOGGER.info(f\"Epoch: {epoch+1} - Accuracy: {accuracy}\")\n        print(classification_result)\n        \n        if accuracy > best_score:\n            best_score = accuracy\n            LOGGER.info(f\"Epoch: {epoch+1} - Save best score: {best_score:.4f} Model\")\n            torch.save({\n                \"model\": model.state_dict(),\n                \"preds\": valid_preds\n            }, str(MODELS_DIR / f\"{Config.model_name}_fold_{fold}_best.pth\"))\n            \n    check_point = torch.load(str(MODELS_DIR / f\"{Config.model_name}_fold_{fold}_best.pth\"))\n    valid_folds[[str(c) for c in range(5)]] = check_point[\"preds\"]\n    valid_folds[\"preds\"] = check_point[\"preds\"].argmax(1)\n    return valid_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_DIR = Path(\"/kaggle/input/fullmodelaugmentation08676/\")\n\npredict_model = Classifier(Config.model_name, pretrained=False)\nstates = [torch.load(str(MODELS_DIR / f\"{Config.model_name}_fold_{fold}_best.pth\"), map_location=torch.device(device)) for fold in Config.train_fold]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in df_train[\"image_id\"].values]\ntrain_dataset = CassavaDataset(image_paths=train_img_paths, \n                               labels=df_train[\"label\"].values,\n                               transform=_get_train_transforms_without_aug())\ntrain_data_loader = DataLoader(train_dataset, batch_size=128, \n                               shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_probs = inference(predict_model, states, train_data_loader, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gt_labels = df_train[\"label\"].values\nohe_labels = np.zeros((gt_labels.size, gt_labels.max()+1))\n\nohe_labels[np.arange(gt_labels.size), gt_labels] = 1\n\nne_indices = np.where(gt_labels != train_probs.argmax(1))[0]\nne_indices.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[[\"class_0\", \"class_1\", \"class_2\", \"class_3\", \"class_4\"]] = train_probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"pred\"] = train_probs.argmax(1)\ndf_train.loc[df_train.label == 4, \"label\"] = df_train[df_train.label == 4][\"pred\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Config.n_epochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_DIR = Path(\"models\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_result(df):\n    preds = df[\"preds\"].values\n    labels = df[\"label\"].values\n    score = accuracy_score(labels, preds)\n    LOGGER.info(f\"Score: {score:<.5f}\")\n\n\nseed_torch(Config.seed)\ndf_folds = create_cv_split(df_train.copy(),\n                           n_splits=Config.n_fold,\n                           shuffle=True,\n                           random_state=Config.seed)\n\n# df_folds = df_folds.sample(100, random_state=Config.seed)\n\noof_df = pd.DataFrame()\n\nfor fold in range(Config.n_fold):\n    if fold == 0:\n        print(\"Skipping first fold...\")\n        continue\n    if fold in Config.train_fold:\n        _oof_df = train_loop(df_folds, fold)\n        oof_df = pd.concat([oof_df, _oof_df])\n        LOGGER.info(f\"====== Fold: {fold} result =======\")\n        get_result(_oof_df)\n        \nLOGGER.info(\"============= CV ===============\")\nget_result(oof_df)\noof_df.to_csv(\"oof_df.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"selam\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODELS_DIR = Path(\"/kaggle/input/fullmodelaugmentation08676/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MODELS_DIR = Path(\"/kaggle/input/full-model-no-augmentation/seresnext_50_32_x4d_full_model_0.8620/\")\n\npredict_model = Classifier(Config.model_name, pretrained=False)\nstates = [torch.load(str(MODELS_DIR / f\"{Config.model_name}_fold_{fold}_best.pth\"), map_location=torch.device(device)) for fold in Config.train_fold]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n\nBASE_TEST_IMGS = \"../input/cassava-leaf-disease-classification/test_images\"\n\ntest_img_paths = [f\"{BASE_TEST_IMGS}/{img_id}\" for img_id in test_df.image_id.values]\n\ntest_dataset = CassavaDataset(image_paths=test_img_paths, transform=get_test_transforms())\ntest_data_loader = DataLoader(test_dataset, \n                             batch_size=Config.batch_size, \n                             shuffle=False)\n\npred_probs = inference(predict_model, states, test_data_loader, device)\nlabels = pred_probs.argmax(1)\n\ntest_df[\"label\"] = labels\n# test_df[[\"image_id\", \"label\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img_paths = [f\"{BASE_TEST_IMGS}/{img_id}\" for img_id in test_df.image_id.values]\n\ntest_dataset = CassavaDataset(image_paths=test_img_paths, transform=get_test_transforms())\ntest_data_loader = DataLoader(test_dataset, \n                             batch_size=Config.batch_size, \n                             shuffle=False)\n\n# run inference 5 times\nfinal_preds = None\nfor j in range(Config.tta_steps):\n    preds = inference(predict_model, states, test_data_loader, device)\n    temp_preds = None\n    for p in preds:\n        if temp_preds is None:\n            temp_preds = p\n        else:\n            temp_preds = np.vstack((temp_preds, p))\n    if final_preds is None:\n        final_preds = temp_preds\n    else:\n        final_preds += temp_preds\nfinal_preds /= 5\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lh models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lh logs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in MODELS_DIR.iterdir():\n    FileLink(file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(\"models/seresnext50_32x4d_fold_4_best.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(\"logs/train.log\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/working/models","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot Augmented Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_img_batch(imgs, labels, rows=3, cols=3):\n    \n    plt.figure(figsize=(16, 12))\n    assert (rows * cols) == len(imgs), \"Flat grid size must be equal to total number of images\"\n    for idx, (img, label) in enumerate(zip(imgs, labels)):\n        plt.subplot(rows, cols, idx + 1)\n        plt.imshow(img)\n        plt.title(f\"Class: {label}\", fontsize=9)\n        plt.axis(\"off\")\n        \n    plt.show()\n    \ndef predict_single_image(model, states, image, device):\n    \n    image = image.to(device)\n    avg_preds = []\n    for state in states:\n        model.load_state_dict(state[\"model\"])\n        model.eval()\n        with torch.no_grad():\n            preds = model(image)\n\n        avg_preds.append(preds.softmax(1).cpu().numpy())\n\n    avg_preds = np.mean(avg_preds, axis=0)    \n    return avg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in df_train[\"image_id\"].values]\ntrain_dataset = CassavaDataset(image_paths=train_img_paths, \n                               labels=df_train[\"label\"].values,\n                               transform=get_train_transforms())\ntrain_data_loader = DataLoader(train_dataset, batch_size=16, \n                               shuffle=False, num_workers=4)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"single_img = read_img_and_cvt_format(train_img_paths[7])\nplt.imshow(single_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_loader = iter(train_data_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = np.array([0.485, 0.456, 0.406])\nstd = np.array([0.229, 0.224, 0.225])\n\nimgs, img_labels = next(train_data_loader)\nreshaped_imgs = imgs.numpy().transpose([0, 2, 3, 1])\nreshaped_imgs = std * reshaped_imgs + mean\n# reshaped_imgs = np.clip(reshaped_imgs, 0, 1)\n\nprintable_labels = [labels[str(current_label.item())] for current_label in img_labels]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_img_batch(reshaped_imgs, printable_labels, 4, 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Error Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img_paths = [f\"{BASE_IMG_DIR}/{img_id}\" for img_id in df_train[\"image_id\"].values]\ntrain_dataset = CassavaDataset(image_paths=train_img_paths, \n                               labels=df_train[\"label\"].values,\n                               transform=_get_train_transforms_without_aug())\ntrain_data_loader = DataLoader(train_dataset, batch_size=128, \n                               shuffle=False, num_workers=4)\n\ntrain_probs = inference(predict_model, states, train_data_loader, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gt_labels = df_train[\"label\"].values\nohe_labels = np.zeros((gt_labels.size, gt_labels.max()+1))\n\nohe_labels[np.arange(gt_labels.size), gt_labels] = 1\n\nne_indices = np.where(gt_labels != train_probs.argmax(1))[0]\nne_indices.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[[\"class_0\", \"class_1\", \"class_2\", \"class_3\", \"class_4\"]] = train_probs\ndf_train[\"pred\"] = train_probs.argmax(1)\ndf_train.loc[df_train.label == 4, \"label\"] = df_train[df_train.label == 4][\"pred\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ((df_train.label != df_train.pred) & (df_train.label == 4)).sum()\n# df_train[(df_train.label != df_train.pred) & (df_train.label == 4)]\n# df_train[df_train.label != df_train.pred].label.value_counts().sort_index() / df_train.label.value_counts().sort_index()\n# df_train[df_train.label == 4].pred.value_counts()\n# df_train[df_train.label == 4].pred.value_counts()\n# df_train[df_train.label == 4].sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# single_img = read_img_and_cvt_format(str(BASE_IMG_DIR / \"1870238448.jpg\"))\n# plt.figure(figsize=(20, 16))\n# plt.imshow(single_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}