{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TPU BiTempered RAdam Cutmix CV5 tf.data TFRec\n\nIn this notebook i'll be exploring the cassav leaf disease detection dataset while studying different techniques for Deep Learning and Dataset pipeline.","metadata":{}},{"cell_type":"code","source":"!pip install -U tensorflow==2.3.2 &> /dev/null\n!pip install --upgrade tensorflow_hub &> /dev/null\n!pip install -U tfa-nightly &> /dev/null\nprint(\"update TPU server tensorflow version...\")\n!pip install cloud-tpu-client &> /dev/null\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:47:33.714410Z","iopub.execute_input":"2021-06-21T15:47:33.714807Z","iopub.status.idle":"2021-06-21T15:48:55.566610Z","shell.execute_reply.started":"2021-06-21T15:47:33.714774Z","shell.execute_reply":"2021-06-21T15:48:55.565230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.backend as K\nimport tensorflow_hub\nimport tensorflow_addons\nimport tensorflow as tf \nimport tensorflow_addons as tfa\nfrom tensorflow_addons.optimizers import RectifiedAdam as RAdam\nfrom tensorflow_addons.optimizers import Lookahead\nfrom cloud_tpu_client import Client\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\nfrom kaggle_datasets import KaggleDatasets\nimport re\nfrom pathlib import Path\nimport os\nimport glob\nimport gc\nfrom functools import partial","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T15:48:55.568956Z","iopub.execute_input":"2021-06-21T15:48:55.569291Z","iopub.status.idle":"2021-06-21T15:48:59.574398Z","shell.execute_reply.started":"2021-06-21T15:48:55.569256Z","shell.execute_reply":"2021-06-21T15:48:59.573645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Configurando TPU kaggle\nClient().configure_tpu_version(tf.__version__, restart_type='ifNeeded')\ngcs_path = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:48:59.575928Z","iopub.execute_input":"2021-06-21T15:48:59.576324Z","iopub.status.idle":"2021-06-21T15:51:06.097301Z","shell.execute_reply.started":"2021-06-21T15:48:59.576295Z","shell.execute_reply":"2021-06-21T15:51:06.096148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:06.098983Z","iopub.execute_input":"2021-06-21T15:51:06.099408Z","iopub.status.idle":"2021-06-21T15:51:13.109649Z","shell.execute_reply.started":"2021-06-21T15:51:06.099377Z","shell.execute_reply":"2021-06-21T15:51:13.108526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.111264Z","iopub.execute_input":"2021-06-21T15:51:13.111716Z","iopub.status.idle":"2021-06-21T15:51:13.117681Z","shell.execute_reply.started":"2021-06-21T15:51:13.111668Z","shell.execute_reply":"2021-06-21T15:51:13.116722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Variáveis globais\nimage_shape = [512,512,3]\nBATCH_SIZE = 128\nAUG_BATCH=BATCH_SIZE\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nfiles = tf.io.gfile.glob(gcs_path+'/train_tfrecords/*.tfrec')\nsteps_per_epoch = int(count_data_items(files)//BATCH_SIZE)\nohe=True\nEPOCHS=40","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.118964Z","iopub.execute_input":"2021-06-21T15:51:13.119265Z","iopub.status.idle":"2021-06-21T15:51:13.216105Z","shell.execute_reply.started":"2021-06-21T15:51:13.119236Z","shell.execute_reply":"2021-06-21T15:51:13.214939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE=[512,512]","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.218047Z","iopub.execute_input":"2021-06-21T15:51:13.218529Z","iopub.status.idle":"2021-06-21T15:51:13.223807Z","shell.execute_reply.started":"2021-06-21T15:51:13.218481Z","shell.execute_reply":"2021-06-21T15:51:13.222634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aumento de Dados <a name=aumento></a>\nMuitas vezes, a tarefa que queremos completar é muito complexo e não temos dados suficientes para treinar nosso modelo. Neste caso, é possível aumentar a qualidade dos nossos dados realizando uma técnica chamada [Data Augmentation](https://journalofbigdata.springeropen.com/articles/10.1186/s40537-019-0197-0). Isso é uma das várias maneiras de mostrar que a qualidade do nosso modelo depende mais dos nossos dados que do modelo em si. Para este modelo, usaremos as seguintes técnicas para a mudança de imagens: Rotação, Shear, Zoom Horizontal e Vertical, Translação Horizontal e Vertical, e duas técnicas avançadas chamadas [Cutmix](https://sarthakforwet.medium.com/cutmix-a-new-strategy-for-data-augmentation-bbc1c3d29aab) e [Mixup](https://paperswithcode.com/method/mixup). Em resumo, Cutmix mistura duas imagens e labels recortando um pedaço de uma imagem e colando por cima de outra, e combinando os labels com proporções ao tamanho do recorte, através da fórmula:\n$$\ny = \\lambda y_i + (1-\\lambda)y_j.\n$$\n\nMixup é similar em questão de combinar diferentes proporções de duas imagens, porém esse mistura é feita em duas imagens inteiras, através da fórmula:\n\n$$\nx = \\lambda x_i + (1-\\lambda)x_j \\\\\ny = \\lambda y_i + (1-\\lambda)y_j.\n$$\nTodas os aumentos são feitas de forma aleatória, com alcance de parâmetros bem definidos. \n\nTensorflow contém várias funções eficientes para o aumento de dados que podem serem usados diretos no pipeline de dados, e muitos podem até serem usados como uma camada, excelente para treino em GPUs. Neste notebook porém, não usaremos destes módulos e implementaremos de forma direta estas funções, pois muitos delas não podem ser usados para treinamento em TPU, e poderemos ter mais controle das distribuíções de valores de transformação (rotacionando aleatoriamente usando uma distribuíção normal). ","metadata":{}},{"cell_type":"markdown","source":"This cell defnies the functions to transform an image in random fashion. It augments the dataset applying: rotation, shearing, zooms and shifts.","metadata":{}},{"cell_type":"code","source":"#Implementação dos aumentos básicos (rotação, sheering, zoom e translação)\n\nROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = np.pi * rotation / 180.\n    shear    = np.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform_mat(image, DIM=IMAGE_SIZE[0]):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 \n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d, [DIM, DIM,3])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.227731Z","iopub.execute_input":"2021-06-21T15:51:13.228089Z","iopub.status.idle":"2021-06-21T15:51:13.249721Z","shell.execute_reply.started":"2021-06-21T15:51:13.228056Z","shell.execute_reply":"2021-06-21T15:51:13.248520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This cell defines the function to apply [mixup](https://towardsdatascience.com/2-reasons-to-use-mixup-when-training-yor-deep-learning-models-58728f15c559).","metadata":{}},{"cell_type":"code","source":"#Aqui está a implementação do mixup\n\ndef mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.251897Z","iopub.execute_input":"2021-06-21T15:51:13.252538Z","iopub.status.idle":"2021-06-21T15:51:13.267134Z","shell.execute_reply.started":"2021-06-21T15:51:13.252487Z","shell.execute_reply":"2021-06-21T15:51:13.266005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This cell defines the function to apply [cutmix](https://sarthakforwet.medium.com/cutmix-a-new-strategy-for-data-augmentation-bbc1c3d29aab).","metadata":{}},{"cell_type":"code","source":"#Aqui está a implementação de cutmix\ndef cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.268866Z","iopub.execute_input":"2021-06-21T15:51:13.269176Z","iopub.status.idle":"2021-06-21T15:51:13.288735Z","shell.execute_reply.started":"2021-06-21T15:51:13.269144Z","shell.execute_reply":"2021-06-21T15:51:13.287374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function is responsible for choosing if it applies mixup or cutmix, or anything at all. It also applies random sturation, contrast, brightness and horizontal flipping.","metadata":{}},{"cell_type":"code","source":"#E aqui está a função que combina todos estes passos. Nós não combinamos cutmix e mixup de uma vez, mas uma proporção SWITCH e (1-SWITCH) de vezes.  De qualquer forma,\n#usaremos uma dessas técnicas 66% das vezes.\ndef transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image1 = []\n    for j in range(AUG_BATCH):\n        img = transform_mat(image[j,])\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n        image1.append(img)\n        \n    image1 = tf.reshape(tf.stack(image1),(AUG_BATCH,DIM,DIM,3))\n    image2, label2 = cutmix(image1, label, CUTMIX_PROB)\n    image3, label3 = mixup(image1, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.290211Z","iopub.execute_input":"2021-06-21T15:51:13.290660Z","iopub.status.idle":"2021-06-21T15:51:13.306529Z","shell.execute_reply.started":"2021-06-21T15:51:13.290628Z","shell.execute_reply":"2021-06-21T15:51:13.305527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This cell defines a function that reads data from tfrecords format, decodes into a jpeg format, than decodes the image into a tensor, while rescaling it form 0-255 to 0-1. ","metadata":{}},{"cell_type":"markdown","source":"O tf.data permite que a gente crie um pipeline de dados para a leitura de dados em disco. Desta forma, não precisaremos de uma memória RAM enorme para o treinamento de um modelo. Porém, há a possibilidade de bottlenecks caso a contrução do pipeline não for feita de maneira correta, fazendo com que tempo seja gasto lendo e processando os dados enquanto o modelo está estagnado esperando mais dados para treinamento. Para mais informações sobre tf.data, leia [esta guia](https://www.tensorflow.org/guide/data).\n\n","metadata":{}},{"cell_type":"code","source":"#Esta função define o processo de leitura de imagem. Primeiro ela decodifica o formato jpeg, depois transforma os dados em float, \n#e após isso redimensiona a imagem para o tamanho escolhido\ndef decode_img(img):\n    image = tf.image.decode_jpeg(img, channels=3)\n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [512,512,3])\n    return image\n#Está função recebe um example tfrecord e retorna a imagem e label, para o caso de treinamento, ou imagem e nome de imagem, para o caso em que estamos na fase de teste.\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_img(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.307543Z","iopub.execute_input":"2021-06-21T15:51:13.307851Z","iopub.status.idle":"2021-06-21T15:51:13.325103Z","shell.execute_reply.started":"2021-06-21T15:51:13.307824Z","shell.execute_reply":"2021-06-21T15:51:13.323800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Está função transforma nossos labels em codificação númerica (1-5) para codificação one-hot (e.g. 0,0,1,0,0). Desta forma poderá ser usados as técnicas MixUp e CutMix.\ndef onehot(image,label):\n    CLASSES = 5\n    return image,tf.one_hot(label,CLASSES)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.326392Z","iopub.execute_input":"2021-06-21T15:51:13.326902Z","iopub.status.idle":"2021-06-21T15:51:13.342491Z","shell.execute_reply.started":"2021-06-21T15:51:13.326860Z","shell.execute_reply":"2021-06-21T15:51:13.341544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Está função recebe uma lista de caminhos para os tfrecords e cria um dataset.\ndef create_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset\n\n\n\n#Dataset for unlabeled images (test set)\ndef inference_dataset(files,number_test_files, return_ids):\n    \n    ds = create_dataset(files, labeled=False, ordered=True)\n    ds = ds.batch(number_test_files)\n    if return_ids:\n        ds = ds.map(lambda img, ids: ids)\n    else:\n        ds = ds.map(lambda img, ids: img)\n    \n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.343781Z","iopub.execute_input":"2021-06-21T15:51:13.344065Z","iopub.status.idle":"2021-06-21T15:51:13.355235Z","shell.execute_reply.started":"2021-06-21T15:51:13.344038Z","shell.execute_reply":"2021-06-21T15:51:13.354282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cria um dataset com a função acima e aplica as transformações, depois embaralha as imagens, junta as imagens em batchs, e usa prefetch para a otimização de pipeline. O prefetch otimiza o pipeline processando batchs de imagens enquanto o modelo está treinando com um batch já recebido. Sem este processo, o modelo teria que esperar o cpu processar os dados toda vez que ela terminar de processar os dados recebidos. Nota-se que usamos o batch antes de aplicar o agumentação de dados, pois o cutmix e mixup precisam de um batch de dados para serem usados.","metadata":{}},{"cell_type":"code","source":"def training_dataset(files):\n    ds = create_dataset(files)\n    ds = ds.cache()\n    ds = ds.repeat()\n    ds = ds.batch(BATCH_SIZE)\n    ds = ds.map(transform, num_parallel_calls=AUTOTUNE)\n    ds = ds.unbatch()\n    ds = ds.shuffle(2048)\n    ds = ds.batch(BATCH_SIZE)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.356922Z","iopub.execute_input":"2021-06-21T15:51:13.357350Z","iopub.status.idle":"2021-06-21T15:51:13.369639Z","shell.execute_reply.started":"2021-06-21T15:51:13.357307Z","shell.execute_reply":"2021-06-21T15:51:13.368603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Este conjunto de dados se aplica ao set de dados para validação. Um processo comum ao Machine Learning em geral é separar nossos dados em treino e validação. Desta forma teremos garantia de que nosso modelo está generalizando para dados fora do conjunto de treinamento, o que é o objetivo principal de machine learning. Notamos que os dados de validação não são aumentados, pois isso só atrapalharia a acurácia do modelo. Existe porém, uma técnica chamada Test Time Augmentation (TTA), onde a imagem é transformada em N diferentes levementes modificadas, para em seguida agregar a previsão de todos elas. ","metadata":{}},{"cell_type":"code","source":"#Notamos que não usamos agumentação de dados. Queremos saber se nosso modelo generaliza para dados que iremos usar em prática.\ndef validation_dataset(files):\n    ds = create_dataset(files)\n    ds = ds.batch(BATCH_SIZE)\n    ds = ds.map(onehot, num_parallel_calls=AUTOTUNE)\n    ds = ds.cache()\n    ds = ds.prefetch(AUTOTUNE)\n    \n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:51:13.371257Z","iopub.execute_input":"2021-06-21T15:51:13.371672Z","iopub.status.idle":"2021-06-21T15:51:13.380922Z","shell.execute_reply.started":"2021-06-21T15:51:13.371638Z","shell.execute_reply":"2021-06-21T15:51:13.379876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Abaixo está a demonstração das imagens modificadas que o modelo irá receber. Podemos ver demonstrações de cutmix e mixup.","metadata":{}},{"cell_type":"code","source":"ds = training_dataset(files).unbatch()\n\n\n\n\n\nfig, axs = plt.subplots(3,3,figsize=(12,12))\nfor i,item in enumerate(ds.take(9)):\n    axs.flat[i].imshow(item[0])\n    axs.flat[i].axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:53:24.034256Z","iopub.execute_input":"2021-06-21T15:53:24.034700Z","iopub.status.idle":"2021-06-21T15:54:42.537051Z","shell.execute_reply.started":"2021-06-21T15:53:24.034664Z","shell.execute_reply":"2021-06-21T15:54:42.536186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = validation_dataset(files).unbatch()\n\n\n\n\n\nfig, axs = plt.subplots(3,3,figsize=(12,12))\nfor i,item in enumerate(ds.take(9)):\n    axs.flat[i].imshow(item[0])\n    axs.flat[i].axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:52:33.692156Z","iopub.execute_input":"2021-06-21T15:52:33.692494Z","iopub.status.idle":"2021-06-21T15:52:39.700090Z","shell.execute_reply.started":"2021-06-21T15:52:33.692438Z","shell.execute_reply":"2021-06-21T15:52:39.699043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.copy('../input/bi-temepered-loss/loss.py', './')\nfrom loss import bi_tempered_logistic_loss\n\nT_1 = 0.8\nT_2 = 1.2\nSMOOTH_FRACTION = 0.1\nN_ITER = 5\n\nwith strategy.scope():\n    class BiTemperedLogisticLoss(tf.keras.losses.Loss):\n        def __init__(self, t1=T_1, t2=T_2, lbl_smth=SMOOTH_FRACTION, n_iter=5):\n          super(BiTemperedLogisticLoss, self).__init__()\n          self.t1 = t1\n          self.t2 = t2\n          self.lbl_smth = lbl_smth\n          self.n_iter = n_iter\n\n        def call(self, y_true, y_pred):\n          return bi_tempered_logistic_loss(y_pred, y_true, self.t1, self.t2, self.lbl_smth, self.n_iter)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:55:02.623135Z","iopub.execute_input":"2021-06-21T15:55:02.623849Z","iopub.status.idle":"2021-06-21T15:55:02.651152Z","shell.execute_reply.started":"2021-06-21T15:55:02.623812Z","shell.execute_reply":"2021-06-21T15:55:02.650331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n# pylint: disable=invalid-name\n\"\"\"EfficientNet models for Keras.\n\nReference paper:\n  - [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks]\n    (https://arxiv.org/abs/1905.11946) (ICML 2019)\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport copy\nimport math\nimport os\n\nfrom tensorflow.python.keras import backend\nfrom tensorflow.python.keras import layers\nfrom tensorflow.python.keras.applications import imagenet_utils\nfrom tensorflow.python.keras.engine import training\nfrom tensorflow.python.keras.utils import data_utils\nfrom tensorflow.python.keras.utils import layer_utils\nfrom tensorflow.python.util.tf_export import keras_export\n\n\nBASE_WEIGHTS_PATH = 'https://storage.googleapis.com/keras-applications/'\n\nWEIGHTS_HASHES = {\n    'b0': ('902e53a9f72be733fc0bcb005b3ebbac',\n           '50bc09e76180e00e4465e1a485ddc09d'),\n    'b1': ('1d254153d4ab51201f1646940f018540',\n           '74c4e6b3e1f6a1eea24c589628592432'),\n    'b2': ('b15cce36ff4dcbd00b6dd88e7857a6ad',\n           '111f8e2ac8aa800a7a99e3239f7bfb39'),\n    'b3': ('ffd1fdc53d0ce67064dc6a9c7960ede0',\n           'af6d107764bb5b1abb91932881670226'),\n    'b4': ('18c95ad55216b8f92d7e70b3a046e2fc',\n           'ebc24e6d6c33eaebbd558eafbeedf1ba'),\n    'b5': ('ace28f2a6363774853a83a0b21b9421a',\n           '38879255a25d3c92d5e44e04ae6cec6f'),\n    'b6': ('165f6e37dce68623721b423839de8be5',\n           '9ecce42647a20130c1f39a5d4cb75743'),\n    'b7': ('8c03f828fec3ef71311cd463b6759d99',\n           'cbcfe4450ddf6f3ad90b1b398090fe4a'),\n}\n\nDEFAULT_BLOCKS_ARGS = [{\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 32,\n    'filters_out': 16,\n    'expand_ratio': 1,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 2,\n    'filters_in': 16,\n    'filters_out': 24,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 2,\n    'filters_in': 24,\n    'filters_out': 40,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 3,\n    'filters_in': 40,\n    'filters_out': 80,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 3,\n    'filters_in': 80,\n    'filters_out': 112,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 4,\n    'filters_in': 112,\n    'filters_out': 192,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 192,\n    'filters_out': 320,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}]\n\nCONV_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 2.0,\n        'mode': 'fan_out',\n        'distribution': 'truncated_normal'\n    }\n}\n\nDENSE_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 1. / 3.,\n        'mode': 'fan_out',\n        'distribution': 'uniform'\n    }\n}\n\n\ndef EfficientNet(\n    width_coefficient,\n    depth_coefficient,\n    default_size,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n    depth_divisor=8,\n    activation='swish',\n    blocks_args='default',\n    model_name='efficientnet',\n    include_top=True,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=None,\n    pooling=None,\n    classes=1000,\n    classifier_activation='softmax',\n):\n  \"\"\"Instantiates the EfficientNet architecture using given scaling coefficients.\n\n  Optionally loads weights pre-trained on ImageNet.\n  Note that the data format convention used by the model is\n  the one specified in your Keras config at `~/.keras/keras.json`.\n\n  Arguments:\n    width_coefficient: float, scaling coefficient for network width.\n    depth_coefficient: float, scaling coefficient for network depth.\n    default_size: integer, default input image size.\n    dropout_rate: float, dropout rate before final classifier layer.\n    drop_connect_rate: float, dropout rate at skip connections.\n    depth_divisor: integer, a unit of network width.\n    activation: activation function.\n    blocks_args: list of dicts, parameters to construct block modules.\n    model_name: string, model name.\n    include_top: whether to include the fully-connected\n        layer at the top of the network.\n    weights: one of `None` (random initialization),\n          'imagenet' (pre-training on ImageNet),\n          or the path to the weights file to be loaded.\n    input_tensor: optional Keras tensor\n        (i.e. output of `layers.Input()`)\n        to use as image input for the model.\n    input_shape: optional shape tuple, only to be specified\n        if `include_top` is False.\n        It should have exactly 3 inputs channels.\n    pooling: optional pooling mode for feature extraction\n        when `include_top` is `False`.\n        - `None` means that the output of the model will be\n            the 4D tensor output of the\n            last convolutional layer.\n        - `avg` means that global average pooling\n            will be applied to the output of the\n            last convolutional layer, and thus\n            the output of the model will be a 2D tensor.\n        - `max` means that global max pooling will\n            be applied.\n    classes: optional number of classes to classify images\n        into, only to be specified if `include_top` is True, and\n        if no `weights` argument is specified.\n    classifier_activation: A `str` or callable. The activation function to use\n        on the \"top\" layer. Ignored unless `include_top=True`. Set\n        `classifier_activation=None` to return the logits of the \"top\" layer.\n\n  Returns:\n    A `keras.Model` instance.\n\n  Raises:\n    ValueError: in case of invalid argument for `weights`,\n      or invalid input shape.\n    ValueError: if `classifier_activation` is not `softmax` or `None` when\n      using a pretrained top layer.\n  \"\"\"\n  if blocks_args == 'default':\n    blocks_args = DEFAULT_BLOCKS_ARGS\n\n  if not (weights in {'imagenet', None} or os.path.exists(weights)):\n    raise ValueError('The `weights` argument should be either '\n                     '`None` (random initialization), `imagenet` '\n                     '(pre-training on ImageNet), '\n                     'or the path to the weights file to be loaded.')\n\n  if weights == 'imagenet' and include_top and classes != 1000:\n    raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                     ' as true, `classes` should be 1000')\n\n  # Determine proper input shape\n  input_shape = imagenet_utils.obtain_input_shape(\n      input_shape,\n      default_size=default_size,\n      min_size=32,\n      data_format=backend.image_data_format(),\n      require_flatten=include_top,\n      weights=weights)\n\n  if input_tensor is None:\n    img_input = layers.Input(shape=input_shape)\n  else:\n    if not backend.is_keras_tensor(input_tensor):\n      img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n    else:\n      img_input = input_tensor\n\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  def round_filters(filters, divisor=depth_divisor):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    filters *= width_coefficient\n    new_filters = max(divisor, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n      new_filters += divisor\n    return int(new_filters)\n\n  def round_repeats(repeats):\n    \"\"\"Round number of repeats based on depth multiplier.\"\"\"\n    return int(math.ceil(depth_coefficient * repeats))\n\n  # Build stem\n  x = img_input\n  #x = layers.Rescaling(1. / 255.)(x)\n  x = layers.Normalization(axis=bn_axis)(x)\n\n  x = layers.ZeroPadding2D(\n      padding=imagenet_utils.correct_pad(x, 3),\n      name='stem_conv_pad')(x)\n  x = layers.Conv2D(\n      round_filters(32),\n      3,\n      strides=2,\n      padding='valid',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='stem_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x)\n  x = layers.Activation(activation, name='stem_activation')(x)\n\n  # Build blocks\n  blocks_args = copy.deepcopy(blocks_args)\n\n  b = 0\n  blocks = float(sum(args['repeats'] for args in blocks_args))\n  for (i, args) in enumerate(blocks_args):\n    assert args['repeats'] > 0\n    # Update block input and output filters based on depth multiplier.\n    args['filters_in'] = round_filters(args['filters_in'])\n    args['filters_out'] = round_filters(args['filters_out'])\n\n    for j in range(round_repeats(args.pop('repeats'))):\n      # The first block needs to take care of stride and filter size increase.\n      if j > 0:\n        args['strides'] = 1\n        args['filters_in'] = args['filters_out']\n      x = block(\n          x,\n          activation,\n          drop_connect_rate * b / blocks,\n          name='block{}{}_'.format(i + 1, chr(j + 97)),\n          **args)\n      b += 1\n\n  # Build top\n  x = layers.Conv2D(\n      round_filters(1280),\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='top_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x)\n  x = layers.Activation(activation, name='top_activation')(x)\n  if include_top:\n    x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    if dropout_rate > 0:\n      x = layers.Dropout(dropout_rate, name='top_dropout')(x)\n    imagenet_utils.validate_activation(classifier_activation, weights)\n    x = layers.Dense(\n        classes,\n        activation=classifier_activation,\n        kernel_initializer=DENSE_KERNEL_INITIALIZER,\n        name='predictions')(x)\n  else:\n    if pooling == 'avg':\n      x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    elif pooling == 'max':\n      x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n  # Ensure that the model takes into account\n  # any potential predecessors of `input_tensor`.\n  if input_tensor is not None:\n    inputs = layer_utils.get_source_inputs(input_tensor)\n  else:\n    inputs = img_input\n\n  # Create model.\n  model = training.Model(inputs, x, name=model_name)\n\n  # Load weights.\n  if weights == 'imagenet':\n    if include_top:\n      file_suffix = '.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][0]\n    else:\n      file_suffix = '_notop.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][1]\n    file_name = model_name + file_suffix\n    weights_path = data_utils.get_file(\n        file_name,\n        BASE_WEIGHTS_PATH + file_name,\n        cache_subdir='models',\n        file_hash=file_hash)\n    model.load_weights(weights_path)\n  elif weights is not None:\n    model.load_weights(weights)\n  return model\n\n\ndef block(inputs,\n          activation='swish',\n          drop_rate=0.,\n          name='',\n          filters_in=32,\n          filters_out=16,\n          kernel_size=3,\n          strides=1,\n          expand_ratio=1,\n          se_ratio=0.,\n          id_skip=True):\n  \"\"\"An inverted residual block.\n\n  Arguments:\n      inputs: input tensor.\n      activation: activation function.\n      drop_rate: float between 0 and 1, fraction of the input units to drop.\n      name: string, block label.\n      filters_in: integer, the number of input filters.\n      filters_out: integer, the number of output filters.\n      kernel_size: integer, the dimension of the convolution window.\n      strides: integer, the stride of the convolution.\n      expand_ratio: integer, scaling coefficient for the input filters.\n      se_ratio: float between 0 and 1, fraction to squeeze the input filters.\n      id_skip: boolean.\n\n  Returns:\n      output tensor for the block.\n  \"\"\"\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  # Expansion phase\n  filters = filters_in * expand_ratio\n  if expand_ratio != 1:\n    x = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        use_bias=False,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'expand_conv')(\n            inputs)\n    x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)\n    x = layers.Activation(activation, name=name + 'expand_activation')(x)\n  else:\n    x = inputs\n\n  # Depthwise Convolution\n  if strides == 2:\n    x = layers.ZeroPadding2D(\n        padding=imagenet_utils.correct_pad(x, kernel_size),\n        name=name + 'dwconv_pad')(x)\n    conv_pad = 'valid'\n  else:\n    conv_pad = 'same'\n  x = layers.DepthwiseConv2D(\n      kernel_size,\n      strides=strides,\n      padding=conv_pad,\n      use_bias=False,\n      depthwise_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'dwconv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)\n  x = layers.Activation(activation, name=name + 'activation')(x)\n\n  # Squeeze and Excitation phase\n  if 0 < se_ratio <= 1:\n    filters_se = max(1, int(filters_in * se_ratio))\n    se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)\n    se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)\n    se = layers.Conv2D(\n        filters_se,\n        1,\n        padding='same',\n        activation=activation,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_reduce')(\n            se)\n    se = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        activation='sigmoid',\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_expand')(se)\n    x = layers.multiply([x, se], name=name + 'se_excite')\n\n  # Output phase\n  x = layers.Conv2D(\n      filters_out,\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'project_conv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)\n  if id_skip and strides == 1 and filters_in == filters_out:\n    if drop_rate > 0:\n      x = layers.Dropout(\n          drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)\n    x = layers.add([x, inputs], name=name + 'add')\n  return x\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB0',\n              'keras.applications.EfficientNetB0')\ndef EfficientNetB0(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.0,\n      224,\n      0.2,\n      model_name='efficientnetb0',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB1',\n              'keras.applications.EfficientNetB1')\ndef EfficientNetB1(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.1,\n      240,\n      0.2,\n      model_name='efficientnetb1',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB2',\n              'keras.applications.EfficientNetB2')\ndef EfficientNetB2(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.1,\n      1.2,\n      260,\n      0.3,\n      model_name='efficientnetb2',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB3',\n              'keras.applications.EfficientNetB3')\ndef EfficientNetB3(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.2,\n      1.4,\n      300,\n      0.3,\n      model_name='efficientnetb3',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB4',\n              'keras.applications.EfficientNetB4')\ndef EfficientNetB4(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.4,\n      1.8,\n      380,\n      0.4,\n      model_name='efficientnetb4',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB5',\n              'keras.applications.EfficientNetB5')\ndef EfficientNetB5(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.6,\n      2.2,\n      456,\n      0.4,\n      model_name='efficientnetb5',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB6',\n              'keras.applications.EfficientNetB6')\ndef EfficientNetB6(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.8,\n      2.6,\n      528,\n      0.5,\n      model_name='efficientnetb6',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB7',\n              'keras.applications.EfficientNetB7')\ndef EfficientNetB7(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      2.0,\n      3.1,\n      600,\n      0.5,\n      model_name='efficientnetb7',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.preprocess_input')\ndef preprocess_input(x, data_format=None):  # pylint: disable=unused-argument\n  return x\n\n\n@keras_export('keras.applications.efficientnet.decode_predictions')\ndef decode_predictions(preds, top=5):\n  \"\"\"Decodes the prediction result from the model.\n\n  Arguments\n    preds: Numpy tensor encoding a batch of predictions.\n    top: Integer, how many top-guesses to return.\n\n  Returns\n    A list of lists of top class prediction tuples\n    `(class_name, class_description, score)`.\n    One list of tuples per sample in batch input.\n\n  Raises\n    ValueError: In case of invalid shape of the `preds` array (must be 2D).\n  \"\"\"\n  return imagenet_utils.decode_predictions(preds, top=top)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-06-21T15:55:03.154619Z","iopub.execute_input":"2021-06-21T15:55:03.155244Z","iopub.status.idle":"2021-06-21T15:55:03.227051Z","shell.execute_reply.started":"2021-06-21T15:55:03.155207Z","shell.execute_reply":"2021-06-21T15:55:03.226177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transferência de Aprendizagem e Definição do Modelo.\nA transferência de aprendizagem é uma tarefa extremamente comum para Deep Learning. A transferência de aprendizagem é quando um modelo treinado em uma tarefa específica é reutilizada de alguma forma para uma outra tarefa. Os benefícios são a rapidez de treinamento, e até a melhora da acurácia, pois alguns modelos e tarefas podem ser complexos demais para nosso conjunto de dados que muitas vezes são pequenas. A transferência de aprendizagem ajuda dar um pontapé no treinamento neste caso. Sabemos que cada camada aprende características de outputs de camadas de níveis mais baixas. Em geral, camadas mais baixas aprendem características generalizadas que aplicam a um número abrangente de tarefas, como por exemplo, a procura de linhas em imagens. A transferência de aprendizagem funciona melhor para tarefas com características dos dados que foram usados para o treinamento forem gerais. Para mais informações sobre, leia [este artigo](https://machinelearningmastery.com/transfer-learning-for-deep-learning/). Em geral, para usar um modelo pré treinado para sua tarefa, retira-se a camada de classificação do modelo anterior e adiciona outras camadas não treinadas para aprender em cima das características das camadas já treinadas.\n\nO modelo usado para esta tarefa foi o EfficientNetB7. Usamos um modleo pré-treinado no conjunto de dados imagenet, famoso por ser usado como benchmark para modelos de classificação de imagens. \n","metadata":{}},{"cell_type":"code","source":"#Model creation. Uses EfficientB6 with global average pooling.\nwith strategy.scope():\n\n    def create_model_eff():\n        base = EfficientNetB7(weights='imagenet',input_shape=image_shape,include_top=False)\n        inputs = keras.layers.Input(shape=image_shape)\n        x = base(inputs)\n        x = keras.layers.GlobalAveragePooling2D()(x)\n        outputs = keras.layers.Dense(5, activation='softmax')(x)\n        \n        model = keras.Model(inputs=inputs, outputs=outputs)\n        return model\n\nwith strategy.scope():\n    model = create_model_eff()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:55:04.957803Z","iopub.execute_input":"2021-06-21T15:55:04.958230Z","iopub.status.idle":"2021-06-21T15:56:17.811742Z","shell.execute_reply.started":"2021-06-21T15:55:04.958178Z","shell.execute_reply":"2021-06-21T15:56:17.810639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Como a camada de output não é inicialmente pré-treinado, é inicialmente perigoso treinar o modelo, pois pode mudar de forma drástica as camadas mais baixas, perdendo grande parte do progresso nos pesos. Há duas alternativas para resolver este problema: Um é congelar todas as camadas do modelo prétreinado e adicionar camadas não treinadas no topo do modelo, e outro é começar com uma taxa de aprendizagem bem baixa para que todas as camadas consigam aprender com a tarefa sem a destruíção dos pesos obtidos. O ultimo será o caso deste modelo. Será aumentado de forma linear até uma certa época, depois será mantida em uma certa taxa, e em seguida diminuída de forma gradativa para o auxílio da convergência do modelo.","metadata":{}},{"cell_type":"code","source":"def get_lr_callback(batch_size=BATCH_SIZE):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:56:17.813704Z","iopub.execute_input":"2021-06-21T15:56:17.814149Z","iopub.status.idle":"2021-06-21T15:56:17.822356Z","shell.execute_reply.started":"2021-06-21T15:56:17.814096Z","shell.execute_reply":"2021-06-21T15:56:17.821208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bagging e Treinamento\n[Bagging](https://en.wikipedia.org/wiki/Bootstrap_aggregating) se refere ao método de criar vários modelos em que o conjunto de dados para o treinamento utilizado para cada um deles é um subconjunto de dados diferente para cada modelo. Desta forma, os modelos aprendem pesos diferentes e as previsões podem ser mais estáveis. Extremamente útil para conjunto de dados com uma variança alta e geralmente tem uma melhora de performance em comparação ao versão de modelo único, porém é computacionalmente mais caro. A versão de bagging usado para este notebook criará 5 modelos, com separação de dados similar ao separação de dados usado em validação cruzada (Cross Validation). A previsão se dará usando a média das previsões de cada modelo e escolhido a classe com a maior pontuação.","metadata":{}},{"cell_type":"code","source":"import gc\n\n#Cria 5 modelos e treina em 5 diferentes folds. Será usado métodos ensemble para a previsão com estes modelos.\ndef CV_models(model_factory,file_paths,n_epochs,n_steps, cv=5 ):\n    \n    print('The current working directory is: ', os.getcwd())\n    \n    \n    folds = KFold(n_splits=cv)\n    histories = []\n    \n    for i, (train_idx, test_idx) in enumerate(folds.split(file_paths)):\n        \n        #Defining training and testing datasets\n        training_files = file_paths[train_idx]\n        test_files = file_paths[test_idx]\n        train_ds = training_dataset(training_files)\n        test_ds = validation_dataset(test_files)\n        \n        #Defining the model file name\n        model_file_name = 'effiecint_b5_fold_{}.hdf5'.format(i)\n        \n        #Utilizamos vários callbacks para o auxílio do treinamento. EarlyStopping para o treinamento quando o loss converge. ModelChecpoint salva o modelo caso algo acontecer e \n        #você não perder o modelo.\n        \n        early_stopping = keras.callbacks.EarlyStopping(patience=2, restore_best_weights=True)\n        checkpoint = keras.callbacks.ModelCheckpoint(model_file_name,save_best_only=True,monitor='val_loss',\\\n                                                     verbose=1)\n        callbacks = [early_stopping,checkpoint, get_lr_callback()]\n        \n        \n        \n        tf.keras.backend.clear_session() \n        with strategy.scope():\n            \n            loss = keras.losses.CategoricalCrossentropy(label_smoothing=0.001)\n            #Um otimizador interessante é o RAdam, ou rectified Adam. É melhor que o Adam, pois é robusto a diferentes taxas de aprendizagem.\n            opt=RAdam()\n            \n            potential_model = Path('/kaggle/input') / 'modelss7' / model_file_name\n            \n            #If model exists in input, \n            if potential_model.exists():\n                print(\"{} exists\".format(potential_model))\n                continue\n                #current_model = tf.keras.models.load_model(potential_model)\n            else:\n                current_model = model_factory() \n                \n            current_model.compile(loss=loss, optimizer=opt, metrics=['categorical_accuracy'])\n            \n        \n        history=current_model.fit(train_ds, epochs=n_epochs, steps_per_epoch=n_steps, callbacks=callbacks, validation_data=test_ds)\n        \n        \n        del current_model\n        del train_ds\n        del test_ds\n        z = gc.collect()\n        histories.append(history)\n            \n        \n            \n        \n    return histories\n","metadata":{"execution":{"iopub.status.busy":"2021-06-21T15:56:17.824593Z","iopub.execute_input":"2021-06-21T15:56:17.825053Z","iopub.status.idle":"2021-06-21T15:56:17.845713Z","shell.execute_reply.started":"2021-06-21T15:56:17.825003Z","shell.execute_reply":"2021-06-21T15:56:17.844373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = CV_models(create_model_eff, np.array(files),EPOCHS,steps_per_epoch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}