{"cells":[{"metadata":{},"cell_type":"markdown","source":"- Thanks for Organizers and congrats to all kagglers. This is my first time of image competition, and I have learned a lot in this competition. I could not get high socre this time, however, I wanted to share my solution for helping anyone to find a new idea. \n\n- My solution is consisted of three parts.  \n **Part 1** - Make a clean model with expected clean data.  \n https://www.kaggle.com/ttkagglett/cassava-cnn-training-with-pre-trained-weights  \n **Part 2** - Make a noise model with clean data and labels that clean model inferenced.  \n https://www.kaggle.com/ttkagglett/cassava-cnn-training-with-noise-labels  \n **Part 3** - Inference with clean model and noise model  \n this notebook. I added a ViT model from public to ensemble them.\n \n- I tried Self-Supervised Learning, however, that does not work for me in this copmpetition.  \n  Notebook is here. If there are any mistakes, please feel free to let me know with comments.  \n  https://www.kaggle.com/ttkagglett/cassava-byol-training\n  \n- Finally, I got some amazing ideas from the following notebook. Thanks.  \n  https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug  \n  https://www.kaggle.com/szuzhangzhi/vision-transformer-vit-cuda-as-usual"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Uninstall fastai for solving dependence problems\n!pip uninstall fastai -y\n# Install packages without internet\n!pip install ../input/packages/torch-1.7.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/packages/torchvision-0.8.2-cp37-cp37m-manylinux1_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/repvggmodels/')\nsys.path.append('../input/vision-transformer-pytorch/VisionTransformer-Pytorch')\n\nfrom repvgg import RepVGG, create_RepVGG_B3g4, create_RepVGG_B3, repvgg_model_convert\nfrom vision_transformer_pytorch import VisionTransformer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport time\nimport random\nimport joblib\nimport sklearn\nimport warnings\nimport multiprocessing\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom glob import glob\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom datetime import datetime\nfrom skimage import io\nfrom sklearn import metrics\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split, GroupKFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom IPython.display import display\nfrom catalyst.data.sampler import BalanceClassSampler\n\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nfrom torch import nn\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom torchvision import models\nfrom torchvision import transforms\n\nfrom albumentations.pytorch import ToTensor, ToTensorV2\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,\n    ShiftScaleRotate, CenterCrop, Resize)\n\npd.set_option(\"max_rows\", 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'seed'       : 42,\n    'fold'       : 0 if len(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))==1 else 99,\n    'tta'        : 1 if len(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))==1 else 4,\n    'img_size'   : 400,\n    'valid_bs'   : 32,\n    'num_workers': multiprocessing.cpu_count(),\n    'device'     : \"cuda:0\" if torch.cuda.is_available() else \"cpu\"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make a model with noise and clean models"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(self, df, data_root, \n                 transforms=None, \n                 output_label=True):\n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms  = transforms\n        self.data_root   = data_root\n        self.output_label  = output_label\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        img = get_img(f\"{self.data_root}/{self.df.loc[index]['image_id']}\")\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FFN(nn.Module):\n    def __init__(self, backbone=None):\n        super(FFN, self).__init__()\n        self.backbone = backbone\n        self.lr1      = nn.Linear(1000, 256)\n        self.relu     = nn.ReLU()\n        self.dropout  = nn.Dropout(0.5)\n        self.lr2      = nn.Linear(256, 5)\n        \n    def forward(self, x):\n        x = self.backbone(x)\n        x = self.relu(self.lr1(x))\n        x = self.dropout(x)\n        x = self.lr2(x)\n        return x\n    \nclass ViT(nn.Module):\n    def __init__(self, backbone=None):\n        super(ViT, self).__init__()\n        self.backbone = backbone\n        \n    def forward(self, x):\n        x = self.backbone(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        image_preds = model(imgs)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(df, data_path, model_name, model_path, backbone, load=True):\n    results = np.zeros((len(os.listdir(data_path)), 5))\n    models  = [m for m in os.listdir(model_path) if m.find(\"csv\")==-1]\n    for model_file in models:\n        device  = torch.device(CFG['device'])\n        dataset = CassavaDataset(df, data_path, transforms=get_inference_transforms(), output_label=False)\n\n        data_loader = torch.utils.data.DataLoader(\n            dataset, \n            batch_size =CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False)\n\n        model = model_name(backbone)\n        if load:\n            if CFG['device'] == \"cpu\":\n                model.load_state_dict(torch.load(f\"{model_path}{model_file}\", map_location=\"cpu\"))\n            else:\n                model.load_state_dict(torch.load(f\"{model_path}{model_file}\"))\n        backbone.to(device)\n        model.to(device)\n\n        preds = []\n        with torch.no_grad():\n            for _ in range(CFG['tta']):\n                preds += [(1/CFG['tta'])*inference_one_epoch(model, data_loader, device)]\n        preds    = np.sum(preds, 0)\n        results += preds\n        del model\n        torch.cuda.empty_cache()\n        \n        if CFG['fold'] == 0:\n            return results\n        \n    return results / len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train      = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain_path = '../input/cassava-leaf-disease-classification/train_images/'\nmodel_path = '../input/cassava-models-trained-with-noise-labels/'\nbackbone   = create_RepVGG_B3g4(deploy=True)\n\nresults_from_noise_model = predict(train, train_path, FFN, model_path, backbone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../input/cassava-noised-label-data/'\nbackbone   = create_RepVGG_B3g4(deploy=True)\n\nresults_from_clean_model = predict(train, train_path, FFN, model_path, backbone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_noise = pd.DataFrame(results_from_noise_model, columns=[f\"l{l}_noise\" for l in range(5)])\ndf_clean = pd.DataFrame(results_from_clean_model, columns=[f\"l{l}_clean\" for l in range(5)])\ndf_all_labels = pd.concat([df_noise, df_clean], axis=1)\n\n# Feature engineering\ndf_all_labels[\"pred_noise\"] = np.argmax(results_from_noise_model, 1)\ndf_all_labels[\"pred_clean\"] = np.argmax(results_from_clean_model, 1)\ndf_all_labels[\"diff_noise_proba\"] = np.max(results_from_noise_model, 1) - (results_from_clean_model * np.identity(5)[np.argmax(results_from_noise_model, 1)]).sum(1)\ndf_all_labels[\"diff_clean_proba\"] = np.max(results_from_clean_model, 1) - (results_from_noise_model * np.identity(5)[np.argmax(results_from_clean_model, 1)]).sum(1)\n\nprint(df_all_labels.shape)\nprint(list(train.label[:5]))\ndf_all_labels.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_accuracy(p, l):\n    if p.ndim==2:\n        print(np.sum(np.argmax(p, 1) == np.array(l)) / l.shape[0])\n    else:\n        print(np.sum(p == np.array(l)) / l.shape[0])\n\nshow_accuracy(results_from_noise_model, train.label)\nshow_accuracy(results_from_clean_model, train.label)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I tried three patterns(SVM/Random Forest/LightGBM). The best private socre came from RF."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(np.array(df_all_labels), np.array(train.label),\n                                                    test_size=0.3, random_state=42, stratify=np.array(train.label))\nrfc = RandomForestClassifier(n_estimators=200, max_depth=10, min_samples_leaf=10, random_state=42)\nrfc.fit(X_train, y_train)\npreds_with_rfc = rfc.predict(X_test)\nshow_accuracy(preds_with_rfc, y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path  = '../input/cassava-leaf-disease-classification/test_images/'\ntest = pd.DataFrame()\ntest['image_id'] = list(os.listdir(test_path))\nmodel_path = '../input/cassava-models-trained-with-noise-labels/'\nbackbone   = create_RepVGG_B3g4(deploy=True)\n\nresults_from_noise_model = predict(test, test_path, FFN, model_path, backbone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../input/cassava-noised-label-data/'\nbackbone   = create_RepVGG_B3g4(deploy=True)\n\nresults_from_clean_model = predict(test, test_path, FFN, model_path, backbone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../input/vit-model-1/'\nbackbone   = VisionTransformer.from_name('ViT-B_16', num_classes=5)\nbackbone.load_state_dict(torch.load(model_path+\"/ViT-B_16.pt\"))\n\nCFG[\"img_size\"] = 384\n\nresults_from_vit = predict(test, test_path, ViT, model_path, backbone, load=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expand_ndim2(ar):\n    if ar.ndim==1:\n        return ar.reshape(-1,1)\n    return ar\n\nar_pred_noise = np.argmax(results_from_noise_model, 1)\nar_pred_clean = np.argmax(results_from_clean_model, 1)\nar_diff_noise_proba = np.max(results_from_noise_model, 1) - (results_from_clean_model * np.identity(5)[np.argmax(results_from_noise_model, 1)]).sum(1)\nar_diff_clean_proba = np.max(results_from_clean_model, 1) - (results_from_noise_model * np.identity(5)[np.argmax(results_from_clean_model, 1)]).sum(1)\n\nar_pred_noise = expand_ndim2(ar_pred_noise)\nar_pred_clean = expand_ndim2(ar_pred_clean)\nar_diff_noise_proba = expand_ndim2(ar_diff_noise_proba)\nar_diff_clean_proba = expand_ndim2(ar_diff_clean_proba)\n                             \nar_all_labels = np.hstack([results_from_noise_model, results_from_clean_model, \n                           ar_pred_noise, ar_pred_clean, ar_diff_noise_proba, ar_diff_clean_proba])\n\n# Make final result\npreds_final   = 0.6*rfc.predict_proba(ar_all_labels) + 0.4*results_from_vit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(preds_final, 1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}