{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is an inference script of the models created from a TPU training script.\n\nTraining : \n- The training script is using only TPU and TFRecord . The official tfrecord seems to have some issue, as my models was stuck during training to a local minimal. https://www.kaggle.com/ludovick/baseline-tf-tpu-efficientnet-kfold-training\n- The data augmentation and the TPU code comes from these two notebooks : https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords and https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease\n- the current possible augmentation are : rotation, shift, zoom, shear, crop, hue etc.\n- tfrecord stratified: https://www.kaggle.com/ludovick/cassavatfrecords512x512q100\n- mixed precision is used : bfloat16 for training\n\nInference :\n- use for now a generator to do the prediction on the test data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#! pip install ../input/efficientnet-keras-source-code\nimport sys\nsys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')\n! pip install -e /kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nimport math, re, os\nimport random\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom tensorflow.keras import backend as K\nfrom sklearn.model_selection import train_test_split\nprint(\"Tensorflow version \" + tf.__version__)\nimport efficientnet.tfkeras as efn\nfrom sklearn.metrics import accuracy_score\nfrom collections import Counter\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    DEVICE = \"TPU\"\nexcept:\n    DEVICE = \"notTPU\"\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncfg = {\"smoothing\":0.00,\n       \"arch_fn\":efn,\n      \"name\": [f\"EfficientNetB{i}\" for i in [4,5,6]],\n      \"num_class\":5,\n       \"epochs\":20,\n       \"kfold\":[5,5,5],\n       \"seed\":42,\n       \"verbose\":1,\n      # augmentation\n       \"resize\":512,\n       \"crop_size\":480,\n       \"rotation\":0.0,\n       \"shear\":0.0,\n       \"h-zoom\":5.0,\n       \"w-zoom\":5.0,\n       \"h-shift\":5.0,\n       \"w-shift\":5.0,\n       \n      \"path_models\":\"../input/cassavatpumodelsbaseline/\"\n      }\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n# GCS_PATH = KaggleDatasets().get_gcs_path(\"cassava-leaf-disease-classification\")\n\nREPLICAS =  strategy.num_replicas_in_sync\nFILENAMES = tf.io.gfile.glob(\"../input/cassava-leaf-disease-classification\" + '/test_tfrecords/ld_test*.tfrec')\nBATCH_SIZE = 128 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ['PYTHONHASHSEED']=str(cfg[\"seed\"])\nos.environ['TF_CUDNN_DETERMINISTIC'] = '1'  # new flag present in tf 2.0+\nrandom.seed(cfg[\"seed\"])\nnp.random.seed(cfg[\"seed\"])\ntf.random.set_seed(cfg[\"seed\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled, return_image_name=False):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        if return_image_name:\n            return image, tf.reshape(tf.one_hot([label], depth=cfg[\"num_class\"], axis=-1), [-1]), example[\"image_name\"]\n        return image, tf.reshape(tf.one_hot([label], depth=cfg[\"num_class\"], axis=-1), [-1])\n    idnum = example['image_name']\n    return image, idnum\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False, return_image_name=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled, return_image_name=return_image_name), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(filenames, ordered=True, tta=False):\n    dataset = load_dataset(filenames, labeled=False, ordered=ordered)\n    if tta:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, cfg):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg[\"resize\"]\n    ROT_ = cfg[\"rotation\"]\n    SHR_ = cfg[\"shear\"]\n    HZOOM_ = cfg[\"h-zoom\"]\n    WZOOM_ = cfg[\"w-zoom\"]\n    HSHIFT_ = cfg[\"h-shift\"]\n    WSHIFT_ = cfg[\"w-shift\"]\n    \n    \n    \n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(img, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    global cfg\n\n    img = transform(img, cfg) # \n    img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_hue(img, 0.01)\n    img = tf.image.random_saturation(img, 0.7, 1.3)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n    img = tf.image.resize(img, [cfg[\"resize\"], cfg[\"resize\"]] )\n    return img, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(cfg, name):\n    model_input = tf.keras.Input(shape=(cfg['resize'], cfg['resize'], 3), name='inputs')\n    constructor = getattr(cfg[\"arch_fn\"], name)\n    x = constructor(include_top=False, weights=None, \n                        input_shape=(cfg['resize'], cfg['resize'], 3), \n                        pooling=None)(model_input)\n    x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    x = tf.keras.layers.Dropout(0.25)(x)\n    outputs = tf.keras.layers.Dense(cfg[\"num_class\"], activation='softmax', name=\"outputs\")(x)\n    model = tf.keras.Model(model_input, outputs, name=name+'_{0}'.format(cfg[\"resize\"]))\n    #model.summary()\n    return model\n\ndef compile_new_model(cfg, name):    \n    with strategy.scope():\n        model = get_model(cfg, name)\n\n        losses = tf.keras.losses.CategoricalCrossentropy(label_smoothing = cfg['smoothing'])\n        model.compile(\n            optimizer = tf.keras.optimizers.Adam(lr=1e-3),\n            loss      = losses,\n            metrics   = tf.keras.metrics.CategoricalAccuracy()\n        )\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"JPEG_PATH = \"../input/cassava-leaf-disease-classification/test_images\"\nJPEG_PATH_TR = \"../input/cassava-leaf-disease-classification/train_images\"\n\nimport cv2\nfrom tqdm.notebook import tqdm\ndef load_image(jpeg_path, image_id):\n    img = cv2.imread(os.path.join(jpeg_path, image_id))/255.0\n    img = cv2.resize(img, (cfg[\"resize\"], cfg[\"resize\"]))[:, :, ::-1]\n\n    return img\ndef generator(filepath, paths, batch_size=32):\n    i=0\n    print(len(paths))\n    while i <= len(paths):\n        batch = []\n        for cpt in range(batch_size):\n            if i + cpt >= len(paths):\n                i += batch_size\n                break\n            batch.append(load_image(filepath, paths[i+cpt]))\n            \n        i += batch_size\n        yield np.stack(batch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntr = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_all = []\nfor num_model in range(len(cfg[\"name\"])):\n    preds_model = []\n    for fold in range(cfg[\"kfold\"][num_model]):\n        print(f\"## FOLD: {fold}\")\n        \n        ds_test = generator(JPEG_PATH,submission.image_id.values)#get_test_dataset(FILENAMES, tta=False)\n        #ds_test = generator(JPEG_PATH_TR, tr.image_id.values)\n\n        # BUILD MODEL\n        K.clear_session()\n        with strategy.scope():\n            model = compile_new_model(cfg, cfg['name'][num_model])\n\n        print('Loading best model...')\n        model.load_weights(cfg[\"path_models\"]+ cfg['name'][num_model] + \"/\" + cfg['name'][num_model]+'-fold-%i.h5'%fold)\n\n        # prediction on val\n\n        preds = model.predict(ds_test, verbose=True)\n        preds_model.append(preds)                 \n\n    preds_model = np.stack(preds_model).mean(0)\n    preds_all.append(preds_model)\n    \npreds_all = np.stack(preds_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_all.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nds_test = get_test_dataset(FILENAMES, tta=False)\n\ntest_ids_ds = ds_test.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(BATCH_SIZE))).numpy().astype('U')\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#submission[\"image_id\"] = test_ids\nsubmission[\"label\"] = preds_all.mean(0).argmax(1)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}