{"cells":[{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install --quiet efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-10-10T21:25:09.94291Z","iopub.status.busy":"2020-10-10T21:25:09.94211Z","iopub.status.idle":"2020-10-10T21:25:26.556399Z","shell.execute_reply":"2020-10-10T21:25:26.555607Z"},"papermill":{"duration":16.665386,"end_time":"2020-10-10T21:25:26.556557","exception":false,"start_time":"2020-10-10T21:25:09.891171","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, applications, Sequential, losses, metrics\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport glob\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-10-10T21:25:26.705656Z","iopub.status.busy":"2020-10-10T21:25:26.704813Z","iopub.status.idle":"2020-10-10T21:25:31.882446Z","shell.execute_reply":"2020-10-10T21:25:31.881768Z"},"papermill":{"duration":5.225184,"end_time":"2020-10-10T21:25:31.882571","exception":false,"start_time":"2020-10-10T21:25:26.657387","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-10T21:25:32.028802Z","iopub.status.busy":"2020-10-10T21:25:32.027887Z","iopub.status.idle":"2020-10-10T21:25:32.031045Z","shell.execute_reply":"2020-10-10T21:25:32.030416Z"},"papermill":{"duration":0.044623,"end_time":"2020-10-10T21:25:32.031164","exception":false,"start_time":"2020-10-10T21:25:31.986541","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 3e-5 * REPLICAS\nEPOCHS = 20\nHEIGHT = 512\nWIDTH = 512\nIMAGE_SIZE = [512, 512]\nCHANNELS = 3\nN_CLASSES = 5\nAUG_BATCH = BATCH_SIZE\nES_PATIENCE = 10\nN_FOLDS = 5","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-10T21:25:32.118263Z","iopub.status.busy":"2020-10-10T21:25:32.115381Z","iopub.status.idle":"2020-10-10T21:25:32.677665Z","shell.execute_reply":"2020-10-10T21:25:32.677043Z"},"papermill":{"duration":0.61129,"end_time":"2020-10-10T21:25:32.677805","exception":false,"start_time":"2020-10-10T21:25:32.066515","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\ntrain = pd.read_csv(f'{database_base_path}train.csv')\nprint(f'Train samples: {len(train)}')\n\n# GCS_PATH = KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification') # Original dataset\n# GCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-{HEIGHT}x{WIDTH}') # Only resized\nGCS_PATH = KaggleDatasets().get_gcs_path(f'cassava-leaf-disease-tfrecords-center-{HEIGHT}x{WIDTH}') # Center croped and resized\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec') # Original TFRecords\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n\nprint(f'GCS: train images: {NUM_TRAINING_IMAGES}')\ndisplay(train.head())\n\nCLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle',\n           'Cassava Mosaic Disease', \n           'Healthy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 0.5):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = N_CLASSES\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n        \ndef mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = N_CLASSES\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\ndef transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    CLASSES = N_CLASSES\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-10-10T21:25:32.781506Z","iopub.status.busy":"2020-10-10T21:25:32.777062Z","iopub.status.idle":"2020-10-10T21:25:32.784982Z","shell.execute_reply":"2020-10-10T21:25:32.78432Z"},"papermill":{"duration":0.072304,"end_time":"2020-10-10T21:25:32.785102","exception":false,"start_time":"2020-10-10T21:25:32.712798","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'target': tf.io.FixedLenFeature([], tf.int64), \n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'image_name': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    if labeled:\n        label_or_name = tf.cast(example['target'], tf.int32)\n    else:\n        label_or_name =  example['image_name']\n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef to_one_hot(image, label):\n    CLASSES = N_CLASSES\n    return image, tf.one_hot(label,CLASSES)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, augment=False):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    \n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    else:\n        dataset = dataset.map(to_one_hot, num_parallel_calls=AUTO)\n        dataset = dataset.cache()\n    if repeated:\n        dataset = dataset.repeat()\n    if not ordered:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    if augment:\n        dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-10-10T21:26:06.501284Z","iopub.status.busy":"2020-10-10T21:26:06.500306Z","iopub.status.idle":"2020-10-10T21:26:06.503182Z","shell.execute_reply":"2020-10-10T21:26:06.502614Z"},"papermill":{"duration":0.159056,"end_time":"2020-10-10T21:26:06.50331","exception":false,"start_time":"2020-10-10T21:26:06.344254","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def model_fn(input_shape, N_CLASSES):\n    input_image = L.Input(shape=input_shape, name='input_image')\n    base_model = efn.EfficientNetB3(input_tensor=input_image, \n                                    include_top=False, \n                                    weights='noisy-student', \n                                    pooling='avg')\n\n    model = Sequential([\n                base_model,\n                L.Dropout(.25),\n                L.Dense(N_CLASSES, activation='softmax', name='output')\n            ])\n\n    optimizer = optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=optimizer, \n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n\nidxT = [0, 2, 3, 4, 5, 7, 9, 10, 11, 12, 13, 14]\nidxV = [1, 6, 8]\nfold = 0\n\n\nif tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\nprint(f'\\nFOLD: {fold+1}')\nprint(f'TRAIN: {idxT} VALID: {idxV}')\n\n# Create train and validation sets\nTRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])\nVALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\nnp.random.shuffle(TRAINING_FILENAMES)\nct_train = count_data_items(TRAIN_FILENAMES)\n\n## MODEL\nK.clear_session()\nwith strategy.scope():\n    model = model_fn((None, None, CHANNELS), N_CLASSES)\n\ncallbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.2),\n    EarlyStopping(monitor='val_accuracy', mode='max', patience=ES_PATIENCE, restore_best_weights=True, verbose=1),\n    tf.keras.callbacks.ModelCheckpoint(filepath='efficientnet_'+str(fold)+'_{epoch}.h5', save_best_only=False, save_weights_only=True)    \n]\n\n## TRAIN\nhistory = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True), \n                    validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False), \n                    steps_per_epoch=(ct_train // BATCH_SIZE), \n                    callbacks=callbacks, \n                    epochs=EPOCHS,  \n                    verbose=2).history\n\nprint(f\"Validation Accuracy = {np.max(history['val_accuracy']):.3f}\")\n\nval_accuracies = np.array(history['val_accuracy'])\ntop3_models = val_accuracies.argsort()[-3:][::-1]\ntop3_models+=1\nbest_models = ['./efficientnet_'+str(fold)+'_'+str(i)+'.h5' for i in top3_models]\nall_models = glob.glob(\"./efficientnet_\"+str(fold)+\"*\")\nfor filename in all_models:\n    if filename not in best_models:\n        os.remove(filename)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n\n# best_models = []\n\n# for fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n#     if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n#     print(f'\\nFOLD: {fold+1}')\n#     print(f'TRAIN: {idxT} VALID: {idxV}')\n\n#     # Create train and validation sets\n#     TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])\n#     VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\n#     np.random.shuffle(TRAINING_FILENAMES)\n#     ct_train = count_data_items(TRAIN_FILENAMES)\n    \n#     ## MODEL\n#     K.clear_session()\n#     with strategy.scope():\n#         model = model_fn((None, None, CHANNELS), N_CLASSES)\n    \n#     callbacks = [\n#         tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.2),\n#         EarlyStopping(monitor='val_accuracy', mode='max', patience=ES_PATIENCE, restore_best_weights=True, verbose=1),\n#         tf.keras.callbacks.ModelCheckpoint(filepath='efficientnet_'+str(fold)+'_{epoch}.h5', save_best_only=False, save_weights_only=True)    \n#     ]\n\n#     ## TRAIN\n#     history = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True), \n#                         validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False), \n#                         steps_per_epoch=(ct_train // BATCH_SIZE), \n#                         callbacks=callbacks, \n#                         epochs=EPOCHS,  \n#                         verbose=2).history\n\n#     print(f\"Validation Accuracy = {np.max(history['val_accuracy']):.3f}\")\n    \n#     val_accuracies = np.array(history['val_accuracy'])\n#     top3_models = val_accuracies.argsort()[-3:][::-1]\n#     top3_models+=1\n#     best_models = ['./efficientnet_'+str(fold)+'_'+str(i)+'.h5' for i in top3_models]\n#     all_models = glob.glob(\"./efficientnet_\"+str(fold)+\"*\")\n#     for filename in all_models:\n#         if filename not in best_models:\n#             os.remove(filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_images_labels(dataset, NUM_IMAGES):\n#     labels_ds = dataset.map(lambda image, label: label).unbatch()\n#     labels = next(iter(labels_ds.batch(NUM_IMAGES))).numpy().astype('U')\n#     labels = np.array(labels, dtype=np.float32)\n#     images = dataset.map(lambda image, label: image)\n#     return images, labels\n\n# with strategy.scope():\n#     model = model_fn((None, None, CHANNELS), N_CLASSES)\n    \n# callbacks = [\n#     tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.2),\n#     EarlyStopping(monitor='val_accuracy', mode='max', patience=ES_PATIENCE, restore_best_weights=True, verbose=1),\n#     tf.keras.callbacks.ModelCheckpoint(filepath='efficientnet{epoch}.h5', save_best_only=False, save_weights_only=True)    \n# ]\n\n# TRAIN_FILENAMES, VALID_FILENAMES = train_test_split(TRAINING_FILENAMES, test_size = 0.1)\n# ct_train = count_data_items(TRAIN_FILENAMES)\n\n# valid_data = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False)\n# test_data = get_dataset(test, labeled=True, ordered=True, repeated=False, augment=False)\n\n# history = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True), \n#                         validation_data=valid_data, \n#                         steps_per_epoch=(ct_train // BATCH_SIZE), \n#                         callbacks=callbacks,\n#                         epochs=EPOCHS,  \n#                         verbose=2).history\n\n# print(f\"Validation Accuracy = {np.max(history['val_accuracy']):.3f}\")\n\n# test_images, test_labels = get_images_labels(test_data, count_data_items(test))\n\n# val_accuracies = np.array(history['val_accuracy'])\n# best_models = val_accuracies.argsort()[-2:][::-1]\n# best_models+=1\n# preds = np.zeros(shape=(count_data_items(test), 5))\n# for i in best_models:\n#     model.load_weights('./efficientnet'+str(i)+'.h5')\n#     preds+=model.predict(test_images)\n# score = accuracy_score(np.argmax(preds, axis=1), np.argmax(test_labels, axis=1))\n# print(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=5, shuffle=True, random_state=seed)\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    print(f'\\nFOLD: {fold+1}')\n    print(', '.join([str(x) for x in idxT]))\n    print(', '.join([str(x) for x in idxV]))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}