{"cells":[{"metadata":{},"cell_type":"markdown","source":"# NOTE \n\n\n\nThis notebook is a \"baseline\" training script of using TPU and TF 2.2. By default, it can use EfficientNet but you can extend it to some others architectures.\n\n- The training script is using custom TFRecord created from the notebook https://www.kaggle.com/spidermandance/resize-tfrecords-and-jpeg with IMG_QUALITY = 100 . The official tfrecord seems to have some issue, as my models was stuck during training to a local minimal.\n- The data augmentation and the TPU code comes from these two notebooks : https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords and https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease\n\n- the current possible augmentation are : rotation, shift, zoom, shear, crop, hue etc.\n- tfrecord stratified: https://www.kaggle.com/ludovick/cassavatfrecords512x512q100\n- mixed precision is used : bfloat16 for training\n- inference script : https://www.kaggle.com/ludovick/baseline-tpu-tf-efficientnet-kfold-gpu-inference\n\n# To Do :\n- add more augmentation : mixup/cutmix/grid etc\n- experiment different architecture\n- better hyperparameters\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import math, re, os\nimport random\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom tensorflow.keras import backend as K\nfrom sklearn.model_selection import train_test_split\nprint(\"Tensorflow version \" + tf.__version__)\nimport efficientnet.tfkeras as efn\nfrom sklearn.metrics import accuracy_score\nfrom collections import Counter\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import mixed_precision\npolicy = mixed_precision.experimental.Policy('mixed_bfloat16')\nmixed_precision.experimental.set_policy(policy)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Connect to TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    DEVICE = \"TPU\"\nexcept:\n    DEVICE = \"notTPU\"\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncfg = {\"smoothing\":0.00,\n       \"arch_fn\":efn,\n      \"name\": [f\"EfficientNetB{i}\" for i in range(6,7)],\n      \"num_class\":5,\n       \"epochs\":25,\n       \"kfold\":5,\n       \"seed\":42,\n       \"verbose\":1,\n      # augmentation\n       \"resize\":512,\n       \"crop_size\":500,\n       \"rotation\":180.0,\n       \"shear\":2.0,\n       \"h-zoom\":5.0,\n       \"w-zoom\":5.0,\n       \"h-shift\":5.0,\n       \"w-shift\":5.0\n      }\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path(\"cassavatfrecords512x512q100\")\nREPLICAS =  strategy.num_replicas_in_sync\nFILENAMES = tf.io.gfile.glob(GCS_PATH + '/ld_train*.tfrec')\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\nAUGMENT = {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAMES, len(FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ['PYTHONHASHSEED']=str(cfg[\"seed\"])\nos.environ['TF_CUDNN_DETERMINISTIC'] = '1'  # new flag present in tf 2.0+\nrandom.seed(cfg[\"seed\"])\nnp.random.seed(cfg[\"seed\"])\ntf.random.set_seed(cfg[\"seed\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# load data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled, return_image_name=False):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        if return_image_name:\n            return image, tf.reshape(tf.one_hot([label], depth=cfg[\"num_class\"], axis=-1), [-1]), example[\"image_name\"]\n        return image, tf.reshape(tf.one_hot([label], depth=cfg[\"num_class\"], axis=-1), [-1])\n    idnum = example['image_name']\n    return image, idnum\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False, return_image_name=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled, return_image_name=return_image_name), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset(filenames, return_image_name=False):\n    dataset = load_dataset(filenames, labeled=True, return_image_name=return_image_name)  \n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)  \n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef get_validation_dataset(filenames, ordered=True, return_image_name=False):\n    dataset = load_dataset(filenames, labeled=True, ordered=ordered, return_image_name=return_image_name)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_test_dataset(filenames, ordered=True):\n    dataset = load_dataset(filenames, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# data aug\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, cfg):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg[\"resize\"]\n    ROT_ = cfg[\"rotation\"]\n    SHR_ = cfg[\"shear\"]\n    HZOOM_ = cfg[\"h-zoom\"]\n    WZOOM_ = cfg[\"w-zoom\"]\n    HSHIFT_ = cfg[\"h-shift\"]\n    WSHIFT_ = cfg[\"w-shift\"]\n    \n    \n    \n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(img, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    global cfg\n\n    img = transform(img, cfg) # \n    img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_hue(img, 0.01)\n    img = tf.image.random_saturation(img, 0.8, 1.2)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n    img = tf.image.resize(img, [cfg[\"resize\"], cfg[\"resize\"]] )\n    return img, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef get_model(cfg, name):\n    model_input = tf.keras.Input(shape=(cfg['resize'], cfg['resize'], 3), name='inputs')\n    constructor = getattr(cfg[\"arch_fn\"], name)\n    x = constructor(include_top=False, weights=\"imagenet\", \n                        input_shape=(cfg['resize'], cfg['resize'], 3), \n                        pooling=None)(model_input)\n    x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    outputs = tf.keras.layers.Dense(cfg[\"num_class\"], activation='softmax', name=\"outputs\")(x)\n    model = tf.keras.Model(model_input, outputs, name=name+'_{0}'.format(cfg[\"resize\"]))\n    return model\n\ndef compile_new_model(cfg, name):    \n    with strategy.scope():\n        model = get_model(cfg, name)\n\n        losses = tf.keras.losses.CategoricalCrossentropy(label_smoothing = cfg['smoothing'])\n        model.compile(\n            optimizer = tf.keras.optimizers.Adam(lr=1e-3),\n            loss      = losses,\n            metrics   = tf.keras.metrics.CategoricalAccuracy()\n        )\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TRAINING"},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_val_all = []\noof_target_all = []\nfor num_model in range(len(cfg[\"name\"])):\n    skf = KFold(n_splits=cfg[\"kfold\"],shuffle=True,random_state=cfg[\"seed\"])\n    oof_val = []\n    oof_target = []\n    for fold,(train_idx, val_idx) in enumerate(skf.split(range(len(FILENAMES)))):\n        print(f\"## FOLD: {fold}\")\n        if DEVICE=='TPU':\n            if tpu: \n                tf.tpu.experimental.initialize_tpu_system(tpu)\n\n        # CREATE TRAIN AND VALIDATION SUBSETS\n\n        filenames_tr = [FILENAMES[i] for i in train_idx]\n        filenames_val = [FILENAMES[i] for i in val_idx]\n        print(f\"Training examples : {count_data_items(filenames_tr)} // Validation examples : {count_data_items(filenames_val)} // {val_idx}\")\n        ds_train = get_training_dataset(filenames_tr)\n        ds_val = get_validation_dataset(filenames_val)\n\n\n        # BUILD MODEL\n        K.clear_session()\n        with strategy.scope():\n            model = compile_new_model(cfg, cfg['name'][num_model])\n\n\n            # callbacks\n            checkpoint = tf.keras.callbacks.ModelCheckpoint(cfg['name'][num_model]+'-fold-%i.h5'%fold, monitor='val_categorical_accuracy', verbose=1, save_best_only=True,\n                save_weights_only=True, mode='max', save_freq='epoch')\n            lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, verbose=1, mode='auto',min_lr=1e-6)\n            callbacks = [checkpoint, lr_reducer]\n\n        # TRAIN\n        print('Training...')\n        history = model.fit(ds_train,epochs= cfg[\"epochs\"], callbacks = callbacks, \n            steps_per_epoch=count_data_items(filenames_tr)/BATCH_SIZE//REPLICAS,\n            validation_data=ds_val, verbose=1)\n\n        print('Loading best model...')\n        model.load_weights(cfg['name'][num_model]+'-fold-%i.h5'%fold)\n\n        # prediction on val\n\n        preds_val = model.predict(ds_val)\n        oof_val.append(preds_val)                 \n\n        oof_target.append( np.array([target.numpy().argmax() for img, target in iter(ds_val.unbatch())]) )\n\n    oof_target = np.concatenate(oof_target)\n    oof_val = np.concatenate(oof_val)\n    \n    oof_val_all.append(oof_val)\n    oof_target_all.append(oof_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_target_all = np.stack(oof_target_all).astype(np.float32)\noof_val_all = np.stack(oof_val_all).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_target_all.shape, oof_val_all.shape, np.argmax(oof_val_all[num_model], axis=-1).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for num_model in range(len(cfg[\"name\"])):\n    print(f\" {cfg['name'][num_model]} : {accuracy_score(oof_target_all[num_model], np.argmax(oof_val_all[num_model], axis=-1))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\" Average : {accuracy_score(oof_target_all[0], np.argmax(oof_val_all.mean(0), axis=1))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n\nJPEG_PATH = \"../input/cassava-leaf-disease-classification/test_images\"\ndef load_image(image_id):\n    img = cv2.imread(os.path.join(JPEG_PATH, image_id))/255.0\n    img = cv2.resize(img, (cfg[\"resize\"], cfg[\"resize\"])\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img\ndef generator(paths, batch_size=32):\n    i=0\n    while i < len(paths):\n        batch = []\n        for cpt in range(batch_size)):\n            batch.append(load_image(paths[i+cpt]))\n            if i + cpt >= len(paths):\n                batch = np.stack(batch)\n                i += batch_size\n                yield batch\n                break\n        batch = np.stack(batch)\n        i += batch_size\n        yield batch\n\"\"\"        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#submission = pd.read_csv(\"sample_submission.csv\")\n#paths = submission.image_id.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#predict = model.predict_generator(generator(paths))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#submission[\"label\"] = predict\n#submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}