{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install --quiet /kaggle/input/kerasapplications\n!pip install --quiet /kaggle/input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#!pip install git+https://github.com/qubvel/efficientnet\n# from keras.applications.nasnet import NASNetMobile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install ../input/efficientnet-keras-source-code/repository/qubvel-efficientnet-c993591","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/classification-models-by-qubvel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random, glob\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import Sequential\nimport efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32 * REPLICAS\nHEIGHT = 512\nWIDTH = 512 \nCHANNELS = 3\nN_CLASSES = 5\nTTA_STEPS = 3 # Do TTA if > 0 \nMODEL_NAME = 'EfficientNet4'\nfocal_loss = False\nSWA = False\nlabel_smoothing = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef get_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    name = parts[-1]\n    return name\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\ndef resize_image(image, label):\n#     image = tf.image.random_crop(image, size=[HEIGHT, WIDTH, CHANNELS])\n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, CHANNELS])\n    return image, label\n\ndef process_path(file_path):\n    name = get_name(file_path)\n    img = tf.io.read_file(file_path)\n    img = decode_image(img)\n    return img, name\n\ndef get_dataset(files_path, shuffled=False, tta=False, extension='jpg'):\n    dataset = tf.data.Dataset.list_files(f'{files_path}*{extension}', shuffle=shuffled)\n    dataset = dataset.map(process_path, num_parallel_calls=AUTO)\n    if tta:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.map(resize_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nsubmission = pd.read_csv(f'{database_base_path}sample_submission.csv')\ndisplay(submission.head())\n\nTEST_FILENAMES = tf.io.gfile.glob(f'{database_base_path}test_tfrecords/ld_test*.tfrec')\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'GCS: test: {NUM_TEST_IMAGES}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path_list = glob.glob('../input/b4es512is45epo-1e4lr03cutmix512fc03dropextdata/model*.h5') #'/kaggle/input/cassava-leaf-disease-tpu-tensorflow-training/*.h5'\nmodel_path_list.sort()\n\nprint('Models to predict:')\nprint(*model_path_list, sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import DenseNet121, DenseNet201 , DenseNet169\nfrom tensorflow.keras.applications import vgg16\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import ResNet101 , ResNet101V2 , ResNet152 , ResNet152V2 , ResNet50V2 , ResNet50\nfrom tensorflow.keras.applications import MobileNet , MobileNetV2\nfrom tensorflow.keras.applications import InceptionResNetV2\nimport tensorflow.keras.layers as L\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom classification_models.keras import Classifiers\n\n\ndef get_model_generalized(name,input_shape,N_CLASSES):\n    \n    if name == 'EfficientNet0' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b0_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB0(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet1' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b1_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB1(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape)\n    elif name == 'EfficientNet2' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b2_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB2(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet4' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b4_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB4(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet6' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b6_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB6(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n    elif name == 'EfficientNet5' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b5_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB5(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n\n    elif name == 'EfficientNet7' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b7_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB7(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n        #base_model.trainable = True\n        #for layer in base_model.layers[:-20] :\n        #    layer.trainable = True\n    elif name == 'EfficientNet3' :\n        wg = '../input/efficientnet-keras-noisystudent-weights-b0b7/efficientnet-b3_noisy-student_notop.h5'\n        base_model = efn.EfficientNetB3(weights=wg,\n                                        include_top = False,\n                                        #pooling='avg',\n                                        input_shape=input_shape\n                                       )\n        #base_model.trainable = True\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True        \n            \n    elif name == 'DenseNet201' :\n        wg = '../input/keras-pretrained-imagenet-weights/densenet201_imagenet_1000_no_top.h5'\n        base_model = DenseNet201(weights=wg,include_top=False,input_shape=input_shape)\n    elif name == 'DenseNet169' :\n        wg = '../input/keras-pretrained-imagenet-weights/densenet169_imagenet_1000_no_top.h5'\n        base_model = DenseNet169(weights=wg,include_top=False,input_shape=input_shape)\n        #base_model.trainable = True\n    elif name == 'DenseNet121' :\n        wg = '../input/keras-pretrained-imagenet-weights/densenet121_imagenet_1000_no_top.h5'\n        base_model = DenseNet121(weights=wg,include_top=False,input_shape=input_shape)\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True\n    elif name == 'MobileNet' :\n        wg = '../input/keras-pretrained-imagenet-weights/mobilenet_imagenet_1000_no_top.h5'\n        base_model = MobileNet(weights = wg, include_top=False,input_shape=input_shape)\n    elif name == 'Inception' :\n        wg = '../input/keras-pretrained-imagenet-weights/inceptionv3_imagenet_1000_no_top.h5'\n        base_model = InceptionV3(weights =wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet50' :\n        wg = '../input/keras-pretrained-imagenet-weights/resnet50_imagenet_1000_no_top.h5'\n        base_model = ResNet50(weights = wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet50V2' :\n        base_model = ResNet50V2(weights = wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet101' :\n        wg = '../input/keras-pretrained-imagenet-weights/resnet101_imagenet_1000_no_top.h5'\n        base_model = ResNet101(weights = wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet101V2' :\n        base_model = ResNet101V2(weights =wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet152' :\n        wg = '../input/keras-pretrained-imagenet-weights/resnet152_imagenet_1000_no_top.h5'\n        base_model = ResNet152(weights = wg,include_top=False,input_shape=input_shape)\n    elif name == 'ResNet152V2' :\n        base_model = ResNet152V2(weights = wg,include_top=False,input_shape=input_shape)\n    elif name == 'Incepresnet' :\n        wg = '../input/keras-pretrained-imagenet-weights/inceptionresnetv2_imagenet_1000_no_top.h5'\n        base_model = InceptionResNetV2(weights =wg,include_top=False,input_shape=input_shape) \n        #base_model.trainable = True\n        #for layer in base_model.layers[:-25] :\n        #    layer.trainable = True\n    elif name == 'seresnet18' : \n        model, preprocess_input = Classifiers.get('seresnet18')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnet18_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnet34' : \n        model, preprocess_input = Classifiers.get('seresnet34')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnet34_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnet50' : \n        model, preprocess_input = Classifiers.get('seresnet50')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnet50_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnet101' : \n        model, preprocess_input = Classifiers.get('seresnet101')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnet101_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnet152' : \n        model, preprocess_input = Classifiers.get('seresnet152')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnet152_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnext50' : \n        model, preprocess_input = Classifiers.get('seresnext50')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnext50_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    elif name == 'seresnext101' : \n        model, preprocess_input = Classifiers.get('seresnext101')\n        wg = '../input/keras-pretrained-imagenet-weights/seresnext101_imagenet_1000_no_top.h5'\n        base_model = model(input_shape=input_shape, weights=wg, include_top=False)\n    x = base_model.output\n    x = L.GlobalAveragePooling2D()(x)\n    x = L.Dense(512,activation='relu')(x)\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(256,activation='relu')(x)\n    x = L.Dropout(0.2)(x)\n    predictions = L.Dense(N_CLASSES,activation='softmax')(x)\n    model = Model(inputs = base_model.input, outputs=predictions) \n    \n    '''if focal_loss : \n        loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    elif label_smoothing :\n        loss=CategoricalCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'categorical_cross_entropy'\n    if SWA :\n        opt = tf.keras.optimizers.Adam(lr=1e-5) # roll back\n        opt = tfa.optimizers.SWA(opt)\n    else :\n        opt = 'adam'\n        \n    optimizer = tf.keras.optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss=losses.SparseCategoricalCrossentropy(), \n                  metrics=['sparse_categorical_accuracy'])'''\n    return model\n\nwith strategy.scope():\n    model = get_model_generalized(MODEL_NAME,(None, None, CHANNELS), N_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files_path = f'{database_base_path}/test_images/'\ntest_preds = np.zeros((len(os.listdir(files_path)), N_CLASSES))\n\n\nfor model_path in model_path_list:\n    print(model_path)\n    K.clear_session()\n    model.load_weights(model_path)\n\n    if TTA_STEPS > 0:\n        test_ds = get_dataset(files_path, tta=True)\n        for step in range(TTA_STEPS):\n            print(f'TTA step {step+1}/{TTA_STEPS}')\n            x_test = test_ds.map(lambda image, image_name: image)\n            test_preds += model.predict(x_test) / (TTA_STEPS * len(model_path_list))\n    else:\n        test_ds = get_dataset(files_path, tta=False)\n        x_test = test_ds.map(lambda image, image_name: image)\n        test_preds += model.predict(x_test) / len(model_path_list)\n    \ntest_preds = np.argmax(test_preds, axis=-1)\nimage_names = [img_name.numpy().decode('utf-8') for img, img_name in iter(test_ds.unbatch())]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'image_id': image_names, 'label': test_preds})\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}