{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries","execution_count":null},{"metadata":{"id":"sQ4KcPjzhTYb","outputId":"fff8014f-7827-4d2d-d561-4d76fb72993b","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn\nimport keras\nfrom keras.layers import *\nfrom keras.applications import *\nfrom keras.models import *\nfrom keras.activations import *\nimport random\nimport tensorflow as tf\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Checking TPU and Its cores","execution_count":null},{"metadata":{"id":"giNWWQMMIfY7","outputId":"5ac889fc-8bd4-4cf0-b12f-047305105885","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Getting Paths to tfrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nGCS_PATH = KaggleDatasets().get_gcs_path()\n\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Splitting file names for training and testing","execution_count":null},{"metadata":{"id":"VOL361X2PP7g","trusted":false},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_filenames , valid_filenames = train_test_split(train_filenames , test_size=0.2,shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initializing Some Constants","execution_count":null},{"metadata":{"id":"bbXkVq9zVq4_","trusted":false},"cell_type":"code","source":"BATCH_SIZE = 8 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512,512]\nAUTO = tf.data.experimental.AUTOTUNE\nimSize = 512","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Some Important Functions","execution_count":null},{"metadata":{"id":"ZHKmm6ormn6f","outputId":"747250f1-afa5-4979-d153-b605cf679a88","trusted":false},"cell_type":"code","source":"import re\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    image = tf.image.resize(image, [imSize,imSize])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_saturation(image, 0, 2)\n    # image = tf.image.random_hue(image,0.15)\n    return image, label   \n\ndef get_training_dataset():\n    dataset = load_dataset(train_filenames, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_val_dataset():\n    dataset = load_dataset(valid_filenames, labeled=True)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(train_filenames)\nNUM_TEST_IMAGES = count_data_items(valid_filenames)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images, {} labeled validation images'.format(NUM_TRAINING_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# checking Samples","execution_count":null},{"metadata":{"id":"lsoXTD9lE7yt","outputId":"c7d6b06c-d674-49c1-9851-d5fc3e484f32","trusted":false},"cell_type":"code","source":"for image, label in get_training_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Training data label examples:\", label.numpy())\n# print(\"Test data shapes:\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Installing Some External Libraries","execution_count":null},{"metadata":{"id":"U35iVPGPMSa0","outputId":"8f378b5a-85be-442a-d6f1-a85efe02d354","trusted":false},"cell_type":"code","source":"!pip install efficientnet\n!pip install tensorflow_addons","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Model Training With Focal Loss","execution_count":null},{"metadata":{"id":"s278XNZ0MFZ3","outputId":"ac6f8236-e837-49e3-d38e-b4620f38c2d5","trusted":false},"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model , load_model\nimport efficientnet.tfkeras as efn\nimport math\nimport tensorflow_addons as tfa\n\nEPOCHS = 25\n\ndef focal_loss(gamma=2., alpha=.25):\n  def focal_loss_fixed(y_true, y_pred):\n    pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n    pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n    pt_1 = K.clip(pt_1, 1e-3, .999)\n    pt_0 = K.clip(pt_0, 1e-3, .999)\n\n    return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1))-K.sum((1-alpha) * K.pow( pt_0, gamma) * K.log(1. - pt_0))\n  \n  return focal_loss_fixed\n\ndef get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.75):\n  \n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n\n        progress = float(epoch - num_warmup_steps ) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=0.00004,num_warmup_steps=4,num_training_steps=EPOCHS)\n\nwith strategy.scope():\n    \n    input = Input(shape=(512,512,3))\n\n    base_model = efn.EfficientNetB3(weights=\"imagenet\",include_top=False,input_shape=(512,512,3))\n    base_model.trainable = True\n    \n    output = base_model(input)\n    output = GlobalMaxPooling2D()(output)\n    output = Dense(256)(output)\n    outptu = LeakyReLU(alpha = 0.25)(output)\n    output = Dropout(0.25)(output)\n\n    output = Dense(16,activation=\"relu\")(output)\n    output = Dropout(0.15)(output)\n\n    output = Dense(1,activation=\"sigmoid\")(output)\n    \n    model = Model(input,output)\n    \n    model.compile(\n        optimizer='adam',\n        loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO),\n        metrics=[tf.keras.metrics.AUC()]\n    )\n    model.summary()\n\nmodel.fit(get_training_dataset(),\n          epochs=EPOCHS,\n          verbose=True,\n          steps_per_epoch=NUM_TRAINING_IMAGES // BATCH_SIZE,\n          validation_data=get_val_dataset(),\n          callbacks=[lr_schedule])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing And Submission Time.","execution_count":null},{"metadata":{"id":"tWvzwTCMgmlp","outputId":"e9c019e1-5748-4424-8cca-d960e318ecc8","trusted":false},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames)\nnum_test_images","execution_count":null,"outputs":[]},{"metadata":{"id":"iMjtv5Y2gnTb","trusted":false},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(test_filenames, labeled=False,ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ntest_dataset = get_test_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"B00VbyVogqta","outputId":"cb81fa0d-cfb2-4762-db2a-cd442cd3ff63","trusted":false},"cell_type":"code","source":"print('Computing predictions...')\ntest_images_ds = test_dataset.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds).flatten()\nprint(probabilities)\n\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_dataset.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(num_test_images))).numpy().astype('U') # all in one batch\nnp.savetxt('EfficientNetB3_focal_v1.csv', np.rec.fromarrays([test_ids, probabilities]), fmt=['%s', '%f'], delimiter=',', header='image_name,target', comments='')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In this notebook i have just implemented a simple code to train EfficientNets on TPU with focal Loss.\nSome Improvements that can be done.\n1. Use Other dataset provided by @cdeotte , it contains 70 k images.\n2. Tuning Layers.\n3. I have seen better results from ensemble of EfficientNets , so you can ensembing EfficientNet 0,1,2,3, etc.\n4. On solo model, i have got EfficientNetB5 to perform the best till now. As EfficientNetB7 is quite large in parameters hence wasn't able to train it.\n4. Changing loss to binary_crossentropy will be also provide better results.\n\nIf you like this notebook , then please upvote and share your suggestions in comment.\n\nThank You.","execution_count":null},{"metadata":{"id":"djJ6Yw6kzc3o","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}