{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\n\nFirst of all, I want to thanks DimitreOliveira for this great script: https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-tpu-v2-pods-inference.\nAlso thanks to Chris Deotte for his amazing scripts were I extract some of the functions used in this Kernel.\n\nThis script is based on 2 parts, the first one is the training pipeline. The second part is inference.\n\nThis was done in colab using 3 TPU-V2. I used 3 TPU-V2 because I realized that my experiments were not deterministic, if I run the same code several times the out of folds give different results and the standard deviation is not small. This is mainly because we have a lot of randomness in our model.\n\nTo determine if a change on the model beneffit my CV I average the 3 out of folds, dont know if 3 is enough but is better than 1 haha. This work pretty well and my CV is increasing with new experiments.\n\nHere are the results of the 3 runs\n\n* EXP1_FOLD1_CV: 0.9018\n* EXP1_FOLD2_CV: 0.9023\n* EXP1_FOLD3_CV: 0.8869\n* EXP1_FOLD4_CV: 0.9065\n* EXP1_FOLD5_CV: 0.8960\n* EXP1_OUT_OF_FOLDS_CV: 0.8987\n\n\n\n* EXP2_FOLD1_CV: 0.8993\n* EXP2_FOLD2_CV: 0.9023\n* EXP2_FOLD3_CV: 0.8958\n* EXP2_FOLD4_CV: 0.9056\n* EXP2_FOLD5_CV: 0.8969\n* EXP2_OUT_OF_FOLDS_CV: 0.9000\n\n\n\n* EXP3_FOLD1_CV: 0.8986\n* EXP3_FOLD2_CV: 0.9007\n* EXP3_FOLD3_CV: 0.8885\n* EXP3_FOLD4_CV: 0.9016\n* EXP3_FOLD5_CV: 0.8972\n* EXP3_OUT_OF_FOLDS_CV: 0.8973\n\nIn the inference part I choose the best fold of the 3 experiments, if that is the case we have\n\n* EXP_FOLD1_CV: 0.9018\n* EXP_FOLD2_CV: 0.9023\n* EXP_FOLD3_CV: 0.8958\n* EXP_FOLD4_CV: 0.9065\n* EXP_FOLD5_CV: 0.8972\n* EXP_MEAN_FOLDS_CV: 0.9007\n\nThe folds of the three experiments have the same validation tf records by the way (using same seed for split).\n\nI hope this insights are usefull, out of folds are saved to check blend scores."},{"metadata":{},"cell_type":"markdown","source":"# Training Pipeline In Colab"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install -q efficientnet\n# import os\n# import re\n# import numpy as np\n# import pandas as pd\n# import random\n# import math\n# from sklearn import metrics\n# from sklearn.model_selection import KFold, StratifiedKFold\n# import tensorflow as tf\n# import efficientnet.tfkeras as efn\n# from tensorflow.keras import backend as K\n# import tensorflow_addons as tfa\n\n# try:\n#     # TPU detection. No parameters necessary if TPU_NAME environment variable is\n#     # set: this is always the case on Kaggle.\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#     print('Running on TPU ', tpu.master())\n# except ValueError:\n#     tpu = None\n\n# if tpu:\n#     tf.config.experimental_connect_to_cluster(tpu)\n#     tf.tpu.experimental.initialize_tpu_system(tpu)\n#     strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# else:\n#     # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n#     strategy = tf.distribute.get_strategy()\n\n# print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# # For tf.dataset\n# AUTO = tf.data.experimental.AUTOTUNE\n\n# # Data access\n# GCS_PATH_TRAINING_FILES = 'gs://kds-97ffa76638f61f952661d26d7752f59abdf8f804f1e4794ab6b2e297'\n# GCS_PATH_TRAINING_FILES_2019 = 'gs://kds-e707b2c823f5ab0df2845ed8abcbed536d27e188731770fde8a6a95b'\n\n# # Training filenames directories\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH_TRAINING_FILES + '/*.tfrec')\n# TRAINING_FILENAMES_2019 = tf.io.gfile.glob(GCS_PATH_TRAINING_FILES_2019 + '/*.tfrec')\n\n# # Configuration\n# EPOCHS = 25\n# BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n# AUG_BATCH = BATCH_SIZE\n# IMAGE_SIZE = [512, 512]\n# # Seed\n# SEED = 123\n# # Learning rate\n# LR = 0.0001\n# # Test time augmentation rounds\n# TTA = 10\n# # Verbosity\n# VERBOSE = 2\n# # Number of classes\n# N_CLASSES = 5\n\n# def seed_everything(seed):\n#     random.seed(seed)\n#     np.random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     tf.random.set_seed(seed)\n\n# def cutmix(image, label, PROBABILITY = 1.0):\n#     # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n#     # output - a batch of images with cutmix applied\n#     DIM = IMAGE_SIZE[0]\n#     CLASSES = 5\n    \n#     imgs = []; labs = []\n#     for j in range(AUG_BATCH):\n#         # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n#         P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n#         # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n#         k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n#         # CHOOSE RANDOM LOCATION\n#         x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n#         y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n#         b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n#         WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n#         ya = tf.math.maximum(0,y-WIDTH//2)\n#         yb = tf.math.minimum(DIM,y+WIDTH//2)\n#         xa = tf.math.maximum(0,x-WIDTH//2)\n#         xb = tf.math.minimum(DIM,x+WIDTH//2)\n#         # MAKE CUTMIX IMAGE\n#         one = image[j,ya:yb,0:xa,:]\n#         two = image[k,ya:yb,xa:xb,:]\n#         three = image[j,ya:yb,xb:DIM,:]\n#         middle = tf.concat([one,two,three],axis=1)\n#         img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n#         imgs.append(img)\n#         # MAKE CUTMIX LABEL\n#         a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n#         if len(label.shape)==1:\n#             lab1 = tf.one_hot(label[j],CLASSES)\n#             lab2 = tf.one_hot(label[k],CLASSES)\n#         else:\n#             lab1 = label[j,]\n#             lab2 = label[k,]\n#         labs.append((1-a)*lab1 + a*lab2)\n            \n#     # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n#     image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n#     label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n#     return image2, label2\n\n# def mixup(image, label, PROBABILITY = 1.0):\n#     # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n#     # output - a batch of images with mixup applied\n#     DIM = IMAGE_SIZE[0]\n#     CLASSES = 5\n    \n#     imgs = []; labs = []\n#     for j in range(AUG_BATCH):\n#         # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n#         P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n#         # CHOOSE RANDOM\n#         k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n#         a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n#         # MAKE MIXUP IMAGE\n#         img1 = image[j,]\n#         img2 = image[k,]\n#         imgs.append((1-a)*img1 + a*img2)\n#         # MAKE CUTMIX LABEL\n#         if len(label.shape)==1:\n#             lab1 = tf.one_hot(label[j],CLASSES)\n#             lab2 = tf.one_hot(label[k],CLASSES)\n#         else:\n#             lab1 = label[j,]\n#             lab2 = label[k,]\n#         labs.append((1-a)*lab1 + a*lab2)\n            \n#     # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n#     image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n#     label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n#     return image2, label2\n\n# def data_augment(image, image_name, target):\n#     p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n#     p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n#     p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n#     p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n#     p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n#     p_crop = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n            \n#     # Flips\n#     image = tf.image.random_flip_left_right(image)\n#     image = tf.image.random_flip_up_down(image)\n#     if p_spatial > 0.75:\n#         image = tf.image.transpose(image)\n        \n#     # Rotates\n#     if p_rotate > 0.75:\n#         image = tf.image.rot90(image, k = 3) # rotate 270º\n#     elif p_rotate > 0.5:\n#         image = tf.image.rot90(image, k = 2) # rotate 180º\n#     elif p_rotate > 0.25:\n#         image = tf.image.rot90(image, k = 1) # rotate 90º\n        \n#     # Pixel-level transforms\n#     if p_pixel_1 >= 0.4:\n#         image = tf.image.random_saturation(image, lower = 0.7, upper = 1.3)\n#     if p_pixel_2 >= 0.4:\n#         image = tf.image.random_contrast(image, lower = 0.8, upper = 1.2)\n#     if p_pixel_3 >= 0.4:\n#         image = tf.image.random_brightness(image, max_delta = 0.1)\n        \n#     # Crops\n#     if p_crop > 0.7:\n#         if p_crop > 0.9:\n#             image = tf.image.central_crop(image, central_fraction = 0.7)\n#         elif p_crop > 0.8:\n#             image = tf.image.central_crop(image, central_fraction = 0.8)\n#         else:\n#             image = tf.image.central_crop(image, central_fraction = 0.9)\n#     elif p_crop > 0.4:\n#         crop_size = tf.random.uniform([], int(IMAGE_SIZE[0] * 0.8), IMAGE_SIZE[0], dtype = tf.int32)\n#         image = tf.image.random_crop(image, size = [crop_size, crop_size, 3])\n\n#     image = tf.image.resize(image, size = IMAGE_SIZE)\n#     image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n#     return image, image_name, target\n\n# def transform(image, image_name, target):\n#     # This Function applies both cutmix and mixup\n#     DIM = IMAGE_SIZE[0]\n#     CLASSES = 5\n#     SWITCH = 0.4\n#     CUTMIX_PROB = 0.7\n#     MIXUP_PROB = 0.7\n#     # For SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n#     image2, label2 = cutmix(image, target, CUTMIX_PROB)\n#     image3, label3 = mixup(image, target, MIXUP_PROB)\n#     imgs = []; labs = []\n#     for j in range(AUG_BATCH):\n#         P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n#         imgs.append(P*image2[j,]+(1-P)*image3[j,])\n#         labs.append(P*label2[j,]+(1-P)*label3[j,])\n#     # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n#     image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n#     label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n#     return image4, image_name, label4\n\n# # Function to decode our images (normalize and reshape)\n# def decode_image(image_data):\n#     image = tf.image.decode_jpeg(image_data, channels = 3)\n#     # Resize image to be aligned with the inference phase\n#     image = tf.image.resize(image, IMAGE_SIZE)\n#     # convert image to floats in [0, 1] range\n#     image = tf.cast(image, tf.float32) / 255.0\n#     # explicit size needed for TPU\n#     image = tf.reshape(image, [*IMAGE_SIZE, 3])\n#     return image\n\n# # This function parse our images and also get the target variable\n# def read_labeled_tfrecord(example):\n#     LABELED_TFREC_FORMAT = {\n#         # tf.string means bytestring\n#         \"image\": tf.io.FixedLenFeature([], tf.string),\n#         \"image_name\": tf.io.FixedLenFeature([], tf.string),\n#         # shape [] means single element\n#         \"target\": tf.io.FixedLenFeature([], tf.int64)\n#     }\n#     example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n#     image = decode_image(example['image'])\n#     image_name = example['image_name']\n#     target = tf.one_hot(tf.cast(example['target'], tf.int32), N_CLASSES)\n#     return image, image_name, target\n\n# def load_dataset(filenames, ordered = False):\n#     # Read from TFRecords. For optimal performance, reading from multiple files at once and\n#     # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n#     ignore_order = tf.data.Options()\n#     if not ordered:\n#         # disable order, increase speed\n#         ignore_order.experimental_deterministic = False \n        \n#     # Automatically interleaves reads from multiple files\n#     dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     # Use data as soon as it streams in, rather than in its original order\n#     dataset = dataset.with_options(ignore_order)\n#     # Returns a dataset of (image, image_name, label)\n#     dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n#     return dataset\n\n# def get_training_dataset(filenames, ordered = False):\n#     dataset = load_dataset(filenames, ordered = ordered)\n#     dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n#     # The training dataset must repeat for several epochs\n#     dataset = dataset.repeat()\n#     dataset = dataset.batch(AUG_BATCH)\n#     dataset = dataset.map(transform, num_parallel_calls = AUTO)\n#     dataset = dataset.unbatch()\n#     dataset = dataset.shuffle(2048)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     # Prefetch next batch while training (autotune prefetch buffer size)\n#     dataset = dataset.prefetch(AUTO)\n#     return dataset\n\n# def get_validation_dataset(filenames, ordered = True):\n#     dataset = load_dataset(filenames, ordered = ordered)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     # Using gpu, not enought memory to use cache\n#     # dataset = dataset.cache()\n#     # Prefetch next batch while training (autotune prefetch buffer size)\n#     dataset = dataset.prefetch(AUTO) \n#     return dataset\n\n# def get_val_tta(filenames, ordered = True):\n#     dataset = load_dataset(filenames, ordered = ordered)\n#     dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n#     dataset = dataset.repeat()\n#     dataset = dataset.batch(BATCH_SIZE)\n#     dataset = dataset.prefetch(AUTO)\n#     return dataset\n\n# # Function to count how many photos we have in\n# def count_data_items(filenames):\n#     # The number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n#     n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n#     return np.sum(n)\n\n# NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\n# NUM_TRAINING_IMAGES_2019 = count_data_items(TRAINING_FILENAMES_2019)\n# print(f'Dataset: {NUM_TRAINING_IMAGES} 2020 training images')\n# print(f'Dataset: {NUM_TRAINING_IMAGES_2019} 2019 training images')\n\n# def get_lr_callback():\n#     lr_start   = 0.000001\n#     lr_max     = 0.000003 * BATCH_SIZE\n#     lr_min     = 0.000001\n#     lr_ramp_ep = 1\n#     lr_sus_ep  = 0\n#     lr_decay   = 0.8\n   \n#     def lrfn(epoch):\n#         if epoch < lr_ramp_ep:\n#             lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n#         elif epoch < lr_ramp_ep + lr_sus_ep:\n#             lr = lr_max    \n#         else:\n#             lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n#         return lr\n\n#     lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n#     return lr_callback\n\n# def get_model():\n    \n#     with strategy.scope():\n        \n#         inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n\n#         x = efn.EfficientNetB5(weights = 'noisy-student', include_top = False)(inp)\n#         x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#         x = tf.keras.layers.Dropout(0.2)(x)\n#         output = tf.keras.layers.Dense(N_CLASSES, activation = 'softmax')(x)\n        \n#         model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n\n#         opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n#         model.compile(\n#             optimizer = opt,\n#             loss = [tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.4)],\n#             metrics = [tf.keras.metrics.CategoricalAccuracy()]\n#         )\n\n#         return model\n    \n# def train_and_evaluate(folds = 5):\n#     oof_image_name = []\n#     oof_target = []\n#     oof_prediction = np.zeros((NUM_TRAINING_IMAGES, N_CLASSES))\n#     previous_number_of_files = 0\n#     total_number_of_files = 0\n    \n#     # Seed everything\n#     seed_everything(SEED)\n#     kfold = KFold(folds, shuffle = True, random_state = SEED)\n#     for fold, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n#         if tpu:\n#             tf.tpu.experimental.initialize_tpu_system(tpu)\n#         print('\\n')\n#         print('-'*50)\n#         print(f'Training fold {fold + 1}')\n#         train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind] + TRAINING_FILENAMES_2019, ordered = False)\n#         train_dataset = train_dataset.map(lambda image, image_name, target: (image, target))\n#         val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], ordered = True)\n#         val_dataset_ = val_dataset.map(lambda image, image_name, target: (image, target))\n#         STEPS_PER_EPOCH = count_data_items([TRAINING_FILENAMES[x] for x in trn_ind] + TRAINING_FILENAMES_2019) // BATCH_SIZE\n#         K.clear_session()\n#         model = get_model()\n#         # Model checkpoint\n#         checkpoint = tf.keras.callbacks.ModelCheckpoint(f'EfficientNetB5_EXP1_{fold}_{IMAGE_SIZE[0]}_{SEED}.h5', \n#                                                         monitor = 'val_categorical_accuracy', \n#                                                         verbose = VERBOSE, \n#                                                         save_best_only = True,\n#                                                         save_weights_only = True, \n#                                                         mode = 'max')\n#         history = model.fit(train_dataset,\n#                             steps_per_epoch = STEPS_PER_EPOCH,\n#                             epochs = EPOCHS,\n#                             callbacks = [checkpoint, get_lr_callback()], \n#                             validation_data = val_dataset_,\n#                             verbose = VERBOSE)\n        \n#         # Load weights from the best epoch\n#         model.load_weights(f'EfficientNetB5_EXP1_{fold}_{IMAGE_SIZE[0]}_{SEED}.h5')\n        \n#         number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])\n#         # Get validation real target and image name\n#         image_name = val_dataset.map(lambda image, image_name, target: image_name).unbatch()\n#         target = val_dataset.map(lambda image, image_name, target: target).unbatch()\n#         image_name = next(iter(image_name.batch(number_of_files))).numpy().astype('U')\n#         target = next(iter(target.batch(number_of_files))).numpy()\n#         target = np.argmax(target, axis = -1)\n#         oof_image_name.extend(list(image_name))\n#         oof_target.extend(list(target))\n        \n#         # Validation time augmentation\n#         steps = TTA * number_of_files / BATCH_SIZE\n#         dataset = get_val_tta([TRAINING_FILENAMES[x] for x in val_ind], ordered = True)\n#         image = dataset.map(lambda image, image_name, target: image)\n#         probabilities = model.predict(image, steps = steps)[: TTA * number_of_files]\n#         probabilities = np.mean(probabilities.reshape((number_of_files, TTA, N_CLASSES), order = 'F'), axis = 1)\n#         total_number_of_files += number_of_files\n#         oof_prediction[previous_number_of_files:total_number_of_files] = probabilities\n#         previous_number_of_files += number_of_files\n        \n#         print('\\n')\n#         print('-'*50)\n#         fold_accuracy_score = metrics.accuracy_score(target, np.argmax(probabilities, axis = -1))\n#         print(f'Our fold {fold + 1} accuracy score validation with {TTA} TTA is {fold_accuracy_score}')\n        \n#     print('\\n')\n#     print('-'*50)\n#     oof_accuracy_score = metrics.accuracy_score(oof_target, np.argmax(oof_prediction, axis = -1))\n#     print(f'Our out of folds accuracy score is {oof_accuracy_score}')\n    \n#     # Save the out of folds predictions\n#     print('Saving out of folds to disk...')\n#     oof_dataset = pd.DataFrame({'oof_image_name': oof_image_name, 'oof_target': oof_target, 'oof_prediction': list(oof_prediction)})\n#     oof_dataset.to_csv(f'EfficientNetB5_EXP1_{IMAGE_SIZE[0]}_{SEED}.csv', index = False)\n    \n# train_and_evaluate(folds = 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference Pipeline"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\n!pip install ../input/cassava-models/Keras_Applications-1.0.8-py3-none-any.whl\n!pip install ../input/cassava-models/efficientnet-1.1.0-py3-none-any.whl\nimport efficientnet.tfkeras as efn\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# # Data access\n# GCS_PATH_TRAINING_FILES = KaggleDatasets().get_gcs_path('cassava-leaf-disease-50-tfrecords-center-512x512')\n# GCS_PATH_TRAINING_FILES_2019 = KaggleDatasets().get_gcs_path('cassava-leaf-disease-50-tfrecords-external-512x512')\n\n# # Training filenames directories\n# TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH_TRAINING_FILES + '/*.tfrec')\n# TRAINING_FILENAMES_2019 = tf.io.gfile.glob(GCS_PATH_TRAINING_FILES_2019 + '/*.tfrec')\n\n# Configuration\nEPOCHS = 20\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\n# Seed\nSEED = 123\n# Learning rate\nLR = 0.0001\n# Test time augmentation rounds\nTTA = 10\n# Verbosity\nVERBOSE = 2\n# Number of classes\nN_CLASSES = 5\n\n# Test filenames directory\nTEST_FILENAMES = '../input/cassava-leaf-disease-classification/test_images/*.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, image_name):\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype = tf.float32)\n            \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > 0.75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > 0.75:\n        image = tf.image.rot90(image, k = 3) # rotate 270º\n    elif p_rotate > 0.5:\n        image = tf.image.rot90(image, k = 2) # rotate 180º\n    elif p_rotate > 0.25:\n        image = tf.image.rot90(image, k = 1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= 0.4:\n        image = tf.image.random_saturation(image, lower = 0.7, upper = 1.3)\n    if p_pixel_2 >= 0.4:\n        image = tf.image.random_contrast(image, lower = 0.8, upper = 1.2)\n    if p_pixel_3 >= 0.4:\n        image = tf.image.random_brightness(image, max_delta = 0.1)\n        \n    # Crops\n    if p_crop > 0.7:\n        if p_crop > 0.9:\n            image = tf.image.central_crop(image, central_fraction = 0.7)\n        elif p_crop > 0.8:\n            image = tf.image.central_crop(image, central_fraction = 0.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction = 0.9)\n    elif p_crop > 0.4:\n        crop_size = tf.random.uniform([], int(IMAGE_SIZE[0] * 0.8), IMAGE_SIZE[0], dtype = tf.int32)\n        image = tf.image.random_crop(image, size = [crop_size, crop_size, 3])\n\n    image = tf.image.resize(image, size = IMAGE_SIZE)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image, image_name\n\n# Function to decode our images (normalize and reshape)\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    # Resize image to be aligned with the inference phase\n    image = tf.image.resize(image, IMAGE_SIZE)\n    # convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0\n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef get_image_name(file_path):\n    parts = tf.strings.split(file_path, os.path.sep)\n    image_name = parts[-1]\n    return image_name\n\ndef read_image(file_path):\n    image_name = get_image_name(file_path)\n    image = tf.io.read_file(file_path)\n    image = decode_image(image)\n    return image, image_name\n\ndef get_test_dataset(filenames, tta = False):\n    dataset = tf.data.Dataset.list_files(filenames, shuffle = False)\n    dataset = dataset.map(read_image, num_parallel_calls = AUTO)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    # The test dataset must repeat if we want to predict with test time augmentation\n    if tta:\n        dataset = dataset.repeat() \n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while predicting (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n\nNUM_TESTING_IMAGES = len(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    \n    with strategy.scope():\n        \n        inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n\n        x = efn.EfficientNetB5(weights = None, include_top = False)(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        x = tf.keras.layers.Dropout(0.2)(x)\n        output = tf.keras.layers.Dense(N_CLASSES, activation = 'softmax')(x)\n        \n        model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n\n        opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.4)],\n            metrics = [tf.keras.metrics.CategoricalAccuracy()]\n        )\n\n        return model\n    \ndef inference(model_paths):\n    \n    # Create a numpy array to store predictions\n    prediction = np.zeros((NUM_TESTING_IMAGES, N_CLASSES))\n    \n    print('Extracting test image names...')\n    # Get the test dataset without tta to extract image names\n    test_dataset = get_test_dataset(TEST_FILENAMES, tta = False)\n    image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    image_name = next(iter(image_name.batch(NUM_TESTING_IMAGES))).numpy().astype('U')\n    print('Test image names completed...')\n    \n    for fold, model_path in enumerate(model_paths):\n        print('\\n')\n        print('-'*50)\n        print(f'Predicting fold {fold + 1}')\n        K.clear_session()\n        model = get_model()\n        # Load weights of pretrained model\n        model.load_weights(model_path)\n        \n        # Add 1 to the steps because we only have a sample for public inference\n        steps = TTA * ((NUM_TESTING_IMAGES / BATCH_SIZE) + 1)\n        # Get the test dataset with tta to extract image\n        test_dataset = get_test_dataset(TEST_FILENAMES, tta = True)\n        image = test_dataset.map(lambda image, image_name: image)\n        probabilities = model.predict(image, steps = steps)[: TTA * NUM_TESTING_IMAGES]\n        probabilities = np.mean(probabilities.reshape((NUM_TESTING_IMAGES, TTA, N_CLASSES), order = 'F'), axis = 1)\n        prediction += probabilities / len(model_paths)\n        \n    sub = pd.DataFrame({'image_id': image_name, 'label': np.argmax(prediction, axis = -1)})\n    sub.to_csv('submission.csv', index = False)\n        \n    return image_name, prediction, sub\n        \n# Get pretrained models list for inference\nmodel_paths = glob.glob('../input/cassava-models/*.h5')\nimage_name, prediction, sub = inference(model_paths)\nsub","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}