{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Library"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm\n\nimport random\nimport os\nimport torch\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom torchvision import models as tvmodels\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\nfrom tqdm import tqdm\nimport torch.nn.functional as F\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold\nimport time\nfrom scipy.special import softmax\nimport math\nfrom matplotlib.pyplot import imread\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\nimport numpy as np\nimport cv2\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\nimport time\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Config\n# ====================================================\nDATA_PATH = '../input/cassava-leaf-disease-classification/'\nTRAIN_DIR = DATA_PATH + 'train_images/'\nTEST_DIR = DATA_PATH + 'test_images/'\nMODEL_PATH = '../input/cassavanet-baseline-models/'\n\nN_TTA = 4\n\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\n\nN_CLASSES = 5\n\nMODEL_LIST = [0,1,2,3,4]\n\nIMG_MEAN = [0.485, 0.456, 0.406] #Mean for normalization Transform cassava = [0.4303, 0.4967, 0.3134] imgnet = [0.485, 0.456, 0.406]\nIMG_STD = [0.229, 0.224, 0.225] #STD for normalization Transform cassava = [0.2142, 0.2191, 0.1954] imgnet = [0.229, 0.224, 0.225]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Seed\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nSEED = 1111\nseed_everything(SEED)  \nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nclass CassavaNet(nn.Module):\n    def __init__(self, model_name=None, pretrained=False):\n        super().__init__()\n        self.model_name = model_name\n        if model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=pretrained)\n        else:\n            self.model = timm.create_model(model_name, pretrained=pretrained)\n        if 'efficientnet' in model_name:\n            self.n_features = self.model.classifier.in_features\n            self.model.classifier = nn.Linear(self.n_features, N_CLASSES)\n        elif model_name == 'vit_large_patch16_384' or model_name == 'deit_base_patch16_224' or model_name == 'deit_base_patch16_384':\n            self.n_features = self.model.head.in_features\n            self.model.head = nn.Linear(self.n_features, N_CLASSES)\n        elif 'resnext' in model_name:\n            self.n_features = self.model.fc.in_features\n            self.model.fc = nn.Linear(self.n_features, N_CLASSES)\n        \n    def forward(self, x):\n        return self.model(x)\n    \n    def freeze(self):\n        # To freeze the residual layers\n        for param in self.model.parameters():\n            param.requires_grad = False\n            \n        if 'efficientnet' in self.model_name:\n            for param in self.model.classifier.parameters():\n                param.requires_grad = True\n        elif self.model_name == 'vit_large_patch16_384' or 'deit_base_patch16_224':\n            for param in self.model.head.parameters():\n                param.requires_grad = True\n        elif 'resnext' in self.model_name:\n            for param in self.model.fc.parameters():\n                param.requires_grad = True\n            \n    def unfreeze(self):\n        # Unfreeze all layers\n        for param in self.model.parameters():\n            param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass GetData(Dataset):\n    def __init__(self, Dir, FNames, labels,Type):\n        self.dir = Dir\n        self.fnames = FNames\n        self.lbs = labels\n        self.type = Type\n        \n    def __len__(self):\n        return len(self.fnames)\n\n    def __getitem__(self, index):\n        x = imread(os.path.join(self.dir, self.fnames[index]))\n        if \"train\" in self.type:\n            aug_data = train_transforms(image = x)\n            return aug_data['image'], self.lbs[index]            \n        elif \"valid\" in self.type:\n            aug_data = valid_transforms(image = x)\n            return aug_data['image'], self.lbs[index]\n        elif \"tr-tst\" in self.type:\n            return x, self.lbs[index]\n        elif \"test\" in self.type:\n            return x, self.fnames[index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Augmentation\n# ====================================================\nAug_Norm = A.Normalize(mean=IMG_MEAN, std=IMG_STD, max_pixel_value=255.0, p=1.0)\ntest_aug = Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p = 1.0),\n            A.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.00, always_apply=False, p=1.0),\n            A.RandomCrop(height= HEIGHT, width = WIDTH,always_apply=True, p=1.0),\n            Aug_Norm,\n            ToTensorV2(p=1.0)\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Model Loading\n# ====================================================\nmodels = []\ncount = 0\nfor model_fpath in os.listdir(MODEL_PATH):\n    if count in MODEL_LIST:\n        print(\"Model Loaded:\",model_fpath)\n        model_name_split = model_fpath.split('_f')[0]\n        model = CassavaNet(model_name_split,pretrained = False)\n        info = torch.load(MODEL_PATH + model_fpath,map_location = torch.device(DEVICE))\n        model.load_state_dict(info)\n        models.append(model)\n    count+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame()\nlist_files = os.listdir(TEST_DIR)\nsubmission['image_id'] = pd.Series(list_files)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TTA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# TTA\n# ====================================================\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    name = parts[-1]\n    return name\n\n#Декодирование\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)#三通道的采样\n    image = tf.cast(image, tf.float32) / 255.0##正则化\n    \n#     image = center_crop(image)\n    return image\n\n\ndef center_crop(image):\n    image = tf.reshape(image, [600, 800, CHANNELS]) # 原始尺寸\n    \n    h, w = image.shape[0], image.shape[1]\n    if h > w:\n        image = tf.image.crop_to_bounding_box(image, (h - w) // 2, 0, w, w)#按最小的边进行放缩\n    else:\n        image = tf.image.crop_to_bounding_box(image, 0, (w - h) // 2, h, h)\n        \n    image = tf.image.resize(image, [HEIGHT, WIDTH]) # Expected shape\n    return image\n\ndef resize_image(image, label):\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])\n    return image, label\n\n\ndef process_path(file_path):\n    name = get_name(file_path)\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, name\n\n\ndef get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):\n    dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTO)\n#    if tta:\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.map(resize_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Аугментация (Некоторые из первого ноутбука)\ndef data_augment(image, label):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # 图形反转\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.8), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git\nimport math, os, re, warnings, random, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import Sequential, Model\nimport efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 21\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport re\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nsubmission = pd.read_csv(f'{database_base_path}sample_submission.csv')\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nTEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nmodel_path_list = glob.glob('/kaggle/input/cassava-leaf-disease-training-with-tpu-v2-pods/*.h5')\nmodel_path_list.sort()\nprint(*model_path_list, sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nBATCH_SIZE = 16\ntest_set = GetData(TEST_DIR,submission['image_id'], submission['label'], Type = 'test')\ntest_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=8,pin_memory = True)\nwith torch.no_grad():\n    for i, (images,labels) in enumerate(test_loader):\n        voting = np.zeros((len(models),N_TTA,N_CLASSES))\n        aug_images = np.zeros((N_TTA,CHANNELS,HEIGHT,WIDTH))\n        for aug_no in range(N_TTA):\n            img_np = images.numpy()\n\n            aug_data = test_aug(image = np.reshape(img_np,(600,800,CHANNELS)))\n            aug_images[aug_no,:,:,:] = aug_data['image'].numpy()\n        aug_images = torch.from_numpy(aug_images).to(torch.float32).to(DEVICE)\n        for model_no in range(len(models)):\n            model = models[model_no]\n            model = model.to(DEVICE)\n            model.eval()            \n\n            logits = model(aug_images)\n            voting[model_no,:,:] = F.softmax(logits).cpu().numpy()\n\n        voting = np.sum(voting,axis = 1) / N_TTA\n        voting = np.sum(voting,axis = 0) / len(models)\n\n        label = np.argmax(voting)\n        submission['label'].loc[submission['image_id'] == labels[0]] = label\nprint(time.time()-start_time)\nprint(voting)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import Sequential, Model\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef model_fn(input_shape, N_CLASSES):\n    inputs = L.Input(shape=input_shape, name='input_image')\n    base_model = efn.EfficientNetB4(input_tensor=inputs, \n                                    include_top=False, \n                                    weights=None, \n                                    pooling='avg')##使用efficientnet\n\n    x = L.Dropout(0.4)(base_model.output)\n    output = L.Dense(N_CLASSES, activation='tanh', name='output')(x)\n    model = Model(inputs=inputs, outputs=output)\n\n    return model\n\n\nmodel = model_fn((None, None, CHANNELS), N_CLASSES)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# try:\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#     print(f'Running on TPU {tpu.master()}')\n# except ValueError:\n#     tpu = None\n\n# if tpu:\n#     tf.config.experimental_connect_to_cluster(tpu)\n#     tf.tpu.experimental.initialize_tpu_system(tpu)\n#     strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# else:\n#     strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\n# REPLICAS = strategy.num_replicas_in_sync\n# print(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfiles_path = f'{database_base_path}test_images/'\ntest_size = len(os.listdir(files_path))\ntest_preds = np.zeros((test_size, N_CLASSES))\nTTA_STEPS = 10 # Do TTA if > 0 \n\nfor model_path in model_path_list:\n    print(model_path)\n    K.clear_session()\n    model.load_weights(model_path)\n\n    if TTA_STEPS > 0:\n        test_ds = get_dataset(files_path, tta=True).repeat()\n        ct_steps = TTA_STEPS * ((test_size/BATCH_SIZE) + 1)\n        preds = model.predict(test_ds, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]\n        preds = np.mean(preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)\n        test_preds += preds / len(model_path_list)\n    else:\n        test_ds = get_dataset(files_path, tta=False)\n        x_test = test_ds.map(lambda image, image_name: image)\n        test_preds += model.predict(x_test) / len(model_path_list)\n\ntest_names_ds = get_dataset(files_path)\nimage_names = [img_name.numpy().decode('utf-8') for img, img_name in iter(test_names_ds.unbatch())]\nprint(test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(voting)\nprint(test_preds)\nout=0.5*test_preds+0.5*np.mean(voting, axis=0)\nout = softmax(out).argmax(axis=-1)\nprint(out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'image_id': image_names, 'label': out})\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}