{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 install --quiet albumentations","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"package_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', '../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0',\n    '../input/image-fmix/FMix-master', '../input/pretrainedmodels/pretrainedmodels-0.7.4'\n]\n\nimport sys\nsys.path.extend(package_paths)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss\nimport json\nimport seaborn as sns\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom collections import defaultdict\nimport torch.nn.functional as F\nimport pretrainedmodels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import for albumentations\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {\n    'input_dir': \"/kaggle/input/cassava-leaf-disease-classification\",\n    'train_folder': \"train_images\",\n    'test_folder': \"test_images\",\n    'seed': 42,\n    'model_arch': ['tf_efficientnet_b5_ns', 'tf_efficientnet_b4_ns', 'tf_efficientnet_b4_ns', 'tf_efficientnet_b5_ns', 'resnet200d',\n                    'resnext50_32x4d', 'seresnext50_32x4d', 'seresnext50_32x4d', 'vit_base_patch16_384', 'se_resnext101_32x4d', 'seresnext50_32x4d', 'tf_efficientnet_b5_ns'],\n    'model_paths': ['/kaggle/input/single-model-efficient-net-b5-ns-wo-dropout', '/kaggle/input/single-model-efficientnet-b4-ns-w-dropout',\n                    '/kaggle/input/single-model-efficientnet-b4-ns', '/kaggle/input/single-model-efficientnet-b5-ns',\n                    '/kaggle/input/single-model-resnet200d', '/kaggle/input/single-model-resnext50', '/kaggle/input/single-model-seresnext50', \n                    '/kaggle/input/single-model-seresnext50-wo-dropout', '/kaggle/input/single-model-vision-transformer', '/kaggle/input/single-model-seresnext101',\n                    '/kaggle/input/single-model-seresnext50-kd-and-pl', '/kaggle/input/single-model-efficientnet-b5-ns-kd-and-pl'],\n    'name_model': ['EfficientNet b5 ns wo dropout', 'EfficientNet b4 ns', 'EfficientNet b4 ns wo dropout', 'EfficientNet b5 ns', 'ResNet200d',\n                   'ResNeXt50', 'SEResNeXt50', 'SEResNeXt50 wo dropout', 'Vision Transformer', 'SEResNeXt101', 'SEResNext50 w kd and pl', 'EfficientNet b5 ns kd and pl'],\n    'name_class': [],\n    'img_size': 512,\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'num_classes': 5,\n    'num_folds': 5,\n    'train_bs': 16,\n    'valid_bs': 100,\n    'test_bs': 100,\n    'num_workers': 4,\n    'tta': 1,\n    'm': [0, 4, 10, 6, 1, 8],# 2],\n    'w': [0.49, 0.2266, 0.092, 0.1948, 0.1238]#, 0.003]\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG['name_class'] = ['MyClassifier' + (str(2) if ('dropout' in name) else str(1))  for name in CFG['name_model'][:-3]] + ['MyClassifier3', 'MyClassifier1', 'MyClassifier1']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_df = pd.read_csv(\"/kaggle/input/cassava-leaf-disease-classification/train.csv\")\ninfo_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef rand_bbox(size, lam):\n    W = size[0]\n    H = size[1]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassavaLeafDataset(Dataset):\n    def __init__(\n        self, df, root_dir, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.root_dir = root_dir\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def read_img(self, image_id):\n        path_img = os.path.join(CFG[\"input_dir\"], self.root_dir, str(image_id))\n        img = cv2.imread(path_img)[:,:,::-1]\n        return img\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n        \n        img  = self.read_img(self.df.iloc[index]['image_id'])\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricMonitor:\n    def __init__(self, float_precision=3):\n        self.float_precision = float_precision\n        self.reset()\n\n    def reset(self):\n        self.metrics = defaultdict(lambda: {\"val\": 0, \"count\": 0, \"avg\": 0})\n\n    def update(self, metric_name, val):\n        metric = self.metrics[metric_name]\n\n        metric[\"val\"] += val\n        metric[\"count\"] += 1\n        metric[\"avg\"] = metric[\"val\"] / metric[\"count\"]\n\n    def __str__(self):\n        return \" | \".join(\n            [\n                \"{metric_name}: {avg:.{float_precision}f}\".format(\n                    metric_name=metric_name, avg=metric[\"avg\"], float_precision=self.float_precision\n                )\n                for (metric_name, metric) in self.metrics.items()\n            ]\n        )\n    \n    def __getitem__(self, metric_name):\n        metric = self.metrics[metric_name]\n        return metric[\"avg\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_nothing(image, *args, **kwargs):\n    return {\"image\": image}\n\ndef enable_if(condition, obj):\n    return obj if (condition) else do_nothing\n\ndef get_augmentation(img_size, mode='train'):\n    return A.Compose([\n        enable_if(mode == \"train\", A.RandomResizedCrop(img_size, img_size)),\n        enable_if(mode == \"train\", A.Transpose(p=0.5)),\n        enable_if(mode == \"train\", A.HorizontalFlip(p=0.5)),\n        enable_if(mode == \"train\", A.VerticalFlip(p=0.5)),\n        enable_if(mode == \"train\", A.ShiftScaleRotate(p=0.5)),\n        enable_if(mode == \"train\", A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5)),\n        enable_if(mode == \"train\", A.RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5)),\n        enable_if(mode == \"val\", A.CenterCrop(img_size, img_size, p=1)),\n        enable_if(mode == \"val\", A.Resize(img_size, img_size)),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        enable_if(mode == \"train\", A.CoarseDropout(p=0.5)),\n        enable_if(mode == \"train\", A.Cutout(p=0.5)),\n        ToTensorV2(p=1.0),\n    ])\n\ndef get_inference_augmentation(img_size):\n    return Compose([\n            RandomResizedCrop(img_size, img_size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyClassifier1(nn.Module):\n    def __init__(self, model_arch, num_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        last_linear = None\n        if ('vit' in model_arch):\n            n_features = self.model.head.in_features\n            self.model.head = nn.Sequential(\n                                        nn.Dropout(0.3),\n                                        nn.Linear(n_features, num_classes)\n                                    )\n        elif ('tf_efficientnet' in model_arch):\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Sequential(\n                                        nn.Dropout(0.3),\n                                        nn.Linear(n_features, num_classes)\n                                    )\n        else:\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Sequential(\n                                        nn.Dropout(0.3),\n                                        nn.Linear(n_features, num_classes)\n                                    )\n    def forward(self, x):\n        return self.model(x)\n    \nclass MyClassifier2(nn.Module):\n    def __init__(self, model_arch, num_classes, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        last_linear = None\n        if ('vit' in model_arch):\n            n_features = self.model.head.in_features\n            self.model.head = nn.Linear(n_features, num_classes)\n        elif ('tf_efficientnet' in model_arch):\n            n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(n_features, num_classes)\n        else:\n            n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(n_features, num_classes)\n    def forward(self, x):\n        return self.model(x)\n    \nclass Flatten(nn.Module):\n    def __init__(self):\n        super(Flatten, self).__init__()\n    \n    def forward(self, x):\n        return x.view(x.shape[0], -1)\n\nclass MyClassifier3(nn.Module):\n    def __init__(self, model_arch, num_classes, pretrained=False):\n        super().__init__()\n        self.model = nn.Sequential(\n            *(list(pretrainedmodels.__dict__['se_resnext101_32x4d'](num_classes=1000, pretrained=None).children())[:-2]),\n            nn.AdaptiveAvgPool2d((1, 1)),\n            Flatten(),\n            nn.Dropout(0.3),\n            nn.Linear(2048, num_classes)\n        )\n    def forward(self, x):\n        return self.model(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf_path_model = \"../input/cassava-leaf-disease-1st-place-models/cropnet_mobilenetv3/cropnet\"\nimport tensorflow_hub as hub\nimport tensorflow as tf\n\ndef build_mobilenetv3(img_size=(224, 224), weights=\"../input/cassava-leaf-disease-1st-place-models/cropnet_mobilenetv3/cropnet\"):\n    classifier = hub.KerasLayer(weights)\n    model = tf.keras.Sequential([\n        tf.keras.layers.InputLayer(input_shape=img_size + (3,)),\n        hub.KerasLayer(classifier, trainable=False)\n    ])\n    return model\n\ndef read_preprocess_file(img_path, normalize=False):\n    img = cv2.imread(img_path)[:,:,::-1]\n    if (normalize):\n        img = img/255.0\n    img = img.astype(np.float32)\n    return img.shape, img\n\ndef image_augmentations(image):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    \n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    \n    if p_spatial > 0.75:\n        image = tf.image.transpose(image)\n        \n    if p_rotate > 0.75:\n        image = tf.image.rot90(image, k = 3)\n    elif p_rotate > 0.5:\n        image = tf.image.rot90(image, k = 2)\n    elif p_rotate > 0.25:\n        image = tf.image.rot90(image, k = 1)\n\n    image = tf.image.resize(image, size = IMAGE_SIZE)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image\n\ndef cut_crop_image(processed_img):\n    image = tf.image.central_crop(processed_img, 0.8)\n    image = tf.image.resize(image, (224, 224))\n    return np.expand_dims(image, 0)\n\ndef distribute_unknown(probabilities):\n    return probabilities[:,:-1] + np.expand_dims(probabilities[:,-1]/5, axis=1)\n\ndef multi_predict_tfhublayer(img_path, modelinstance):\n    img = cut_crop_image(read_preprocess_file(img_path, True)[1])\n    yhat = modelinstance.predict(img)\n    return np.mean(distribute_unknown(yhat), axis=0)\n\ndef predict_and_vote(image_list, modelinstances, onlykeras):\n    predictions = [] \n    with tqdm(total=len(image_list)) as process_bar:       \n        for img_path in image_list:\n            process_bar.update(1)  \n            Yhats = np.vstack([func(img_path, modelinstance) for func, modelinstance in modelinstances])\n            if onlykeras:\n                predictions.append(np.argmax(np.sum(Yhats, axis=0)))\n            else:\n                predictions.append(Yhats)\n    predictions = np.concatenate(predictions, axis=0)\n    return predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(test_df, test_dir=CFG[\"test_folder\"]):\n    device = torch.device(CFG[\"device\"])\n    \n    tst_all = []\n    for i, m in enumerate(CFG['m']):\n        model_arch = CFG['model_arch'][m]\n        img_size = 512 if ('vit' not in model_arch) else 384\n        test_transforms = get_augmentation(img_size, mode=\"val\")\n        test_dataset = CassavaLeafDataset(test_df, test_dir, test_transforms, output_label=False)\n        test_loader = DataLoader(test_dataset, batch_size=CFG[\"test_bs\"], shuffle=False, num_workers=CFG[\"num_workers\"])\n        model = globals()[CFG['name_class'][m]](model_arch, CFG['num_classes']).to(device)\n        paths = np.array(glob(CFG['model_paths'][m] + \"/*\"))\n        fold_idx = np.array([int(path.split('fold_')[1][0]) for path in paths])\n        paths = paths[np.argsort(fold_idx)]\n        tst_preds_one_model = []\n        for fold in range(5):\n            model.load_state_dict(torch.load(paths[fold]))\n            for _ in range(1):\n                with torch.no_grad():\n                    img_preds_all = inference_one_epoch(model, test_loader, device)\n                    tst_preds_one_model += [img_preds_all]\n        tst_preds_one_model = np.mean(tst_preds_one_model, axis=0)\n        tst_all += [tst_preds_one_model]\n    final_output = tst_all[0]\n    for k in range(1, len(tst_all)):\n        final_output = (1 - CFG[\"w\"][k-1]) * final_output + CFG[\"w\"][k-1] * tst_all[k]\n    return final_output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pytorch_prob = inference(submission)\nmodel_mobilenet = build_mobilenetv3()\ntensorflow_prob = predict_and_vote([os.path.join(CFG[\"input_dir\"], CFG[\"test_folder\"], id) for id in submission[\"image_id\"].values], [(multi_predict_tfhublayer, model_mobilenet)], onlykeras=False)\nsubmission[\"label\"] = np.argmax(np.mean([pytorch_prob, tensorflow_prob], axis=0), axis=1)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}