{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Installing Tensorflow Addons For Data Augmentation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tfa-nightly","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q pyyaml h5py","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **IMPORT REQUIRED LIBRARIES**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport random \nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Initialize the strategy for TPU if available**"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Running on Tpu\" , tpu.master())\nexcept ValueError as e:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS:\" , strategy.num_replicas_in_sync)\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Importing path for kaggle dataset**"},{"metadata":{},"cell_type":"markdown","source":"Note : Importing directly from the kaggle input dir didn't worked for me on TPU's . Was stuck for sometime "},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Defining Required Variables And Data Format**"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\nLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"class\": tf.io.FixedLenFeature([], tf.int64)}\n\nUNLABELED_TFREC_FORMAT = {\n    \"image\": tf.io.FixedLenFeature([], tf.string), \n    \"id\": tf.io.FixedLenFeature([], tf.string), \n    \n}\nIMAGE_SIZE = [512,512]\nEPOCHS = 20\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Importing The Binary or TFREC**"},{"metadata":{"trusted":true},"cell_type":"code","source":"training_data = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-512x512/train/*.tfrec\")\nvalidation_data = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-512x512/val/*.tfrec\")\ntesting_data = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-512x512/test/*.tfrec\")\nNUM_CLASSES = 104\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Some Data Augmentation (flip , rotate , color shifting)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def flip(x: tf.Tensor) -> tf.Tensor:\n    check = random.randint(0,9)\n    \n    if check < 5:\n        x = tf.image.random_flip_left_right(x)\n        return x\n    \n    x = tf.image.random_flip_up_down(x)\n\n    return x\n\n\n\ndef color(x: tf.Tensor) -> tf.Tensor:\n    x = tf.image.random_hue(x, 0.08)\n    x = tf.image.random_saturation(x, 0.6, 1.6)\n    x = tf.image.random_brightness(x, 0.05)\n    x = tf.image.random_contrast(x, 0.7, 1.3)\n    return x \n\n\n\ndef rotate(x: tf.Tensor) -> tf.Tensor:\n    return tf.image.rot90(x, tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32))\n\n\n\ndef shear(x: tf.Tensor) -> tf.Tensor:\n    check = random.randint(0 , 5)\n    \n    if check > 3:\n        x = tfa.image.shear_x(x , 0.2 , 0)\n        return x\n    \n    x = tfa.image.shear_y(x , 0.2 , 0)\n    \n    return x\n\ndef random_all(x:tf.Tensor) -> tf.Tensor:\n    x = tf.image.random_flip_left_right(x)\n    x = tf.image.random_flip_up_down(x)\n    x = tf.image.random_hue(x, 0.08)\n    x = tf.image.random_saturation(x, 0.6, 1.6)\n    x = tf.image.random_brightness(x, 0.05)\n    x = tf.image.random_contrast(x, 0.7, 1.3)\n    return tf.image.rot90(x, tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Data Augment**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def augment_data(image , label):\n    \n    check = random.randint(0,9)\n    \n    if check == 1:\n        image = flip(image)\n        \n    elif check == 2:\n        image = color(image)\n    \n    elif check == 3:\n        image = rotate(image)\n    \n    elif check == 4:\n        image = shear(image)\n    \n    elif check == 5:\n        image = random_all(image)\n    \n    return image , label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Converting Tensor To jpeg Images So It Can Be Processed And Augmented**"},{"metadata":{},"cell_type":"markdown","source":"Note : Here it randomly augments the data. Augmenting here for some reason didnt converse loss into global minimum. So commented it out"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Reading Labeled tfrec**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(record):\n    record = tf.io.parse_single_example(record , LABELED_TFREC_FORMAT)\n    image = decode_image(record['image'])\n    label = tf.cast(record['class'] , tf.int32)\n    return image , label\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Reading Unlabeled tfrec**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(record):\n    record = tf.io.parse_single_example(record , UNLABELED_TFREC_FORMAT)\n    image = decode_image(record['image'])\n    id_num = record['id'] \n    return image , id_num","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mapping the available dataset to labeled and unlabeled "},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames , labeled=True , ordered = False):\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Mapping data to be augmented**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_augmented_dataset(filenames , labeled=True , ordered = False):\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord)\n    dataset = dataset.map(augment_data)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert training dataset into batches"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(training_data , labeled = True , ordered = False)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Getting Augmented Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_augmented_dataset():\n    dataset = load_augmented_dataset(training_data , labeled = True , ordered = False)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Converting dev/validation dataset into batches**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset():\n    dataset = load_dataset(validation_data , labeled = True , ordered = False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    return dataset ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Getting dataset to be predicted"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered = False):\n    dataset = load_dataset(testing_data , labeled = False , ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Since TensorFlow doesn't provides this classification model yet. We need to install it from third party or PIP**"},{"metadata":{},"cell_type":"markdown","source":"> ****Note :  I experimented with different models like VGG-16 , VGG-19 , ResNet50 , NASNetLarge , Xception  but the best result came from efficientnetB7**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.tfkeras as efn\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Defining Model and Callbacks **"},{"metadata":{},"cell_type":"markdown","source":"**Callbacks Used**\n* Early Stopping so the model doesnt overfits \n* Learning Rate Scheduling as constant learning rate didn't give much accuracy "},{"metadata":{"trusted":true},"cell_type":"code","source":"callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss' , patience = 4)\n\nlearning_rate_start =  0.00001\n\nlearning_rate_max = 0.00005 * strategy.num_replicas_in_sync\n\nlearning_rate_min = 0.0001\n\nlearning_rate_boost_epochs = 3\n\nlearning_rate_sustain_epochs = 0 \n\nlearning_rate_decay = 0.9\n\ndef learning_rate_schedule(epoch):\n    if epoch < learning_rate_boost_epochs:\n        \n        lr = (learning_rate_max - learning_rate_start) / learning_rate_boost_epochs * epoch + learning_rate_start\n        \n    elif epoch < learning_rate_boost_epochs + learning_rate_sustain_epochs:\n        \n        lr = learning_rate_max\n        \n    else:\n        \n        lr = (learning_rate_max - learning_rate_min) * learning_rate_decay **(epoch - learning_rate_boost_epochs - learning_rate_sustain_epochs) + learning_rate_min\n        \n    return lr\n\n\nlearning_rate_callback = tf.keras.callbacks.LearningRateScheduler(learning_rate_schedule , verbose = True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    with strategy.scope():\n        input_layer = tf.keras.layers.Input(shape = (*IMAGE_SIZE,3))\n\n        pretrained_model = efn.EfficientNetB7(include_top = False , weights = 'noisy-student' , input_shape = (*IMAGE_SIZE,3) , input_tensor = input_layer , pooling='avg')   \n\n        for layer in pretrained_model.layers:\n            layer.trainable = True\n\n        X = tf.keras.layers.Dropout(0.2)(pretrained_model.layers[-1].output)\n\n        X = tf.keras.layers.Dense(NUM_CLASSES , activation = 'softmax' , dtype= 'float32')(X)\n\n\n        model = tf.keras.Model(inputs = input_layer , outputs = X)\n\n        optimizer = tf.keras.optimizers.Adam(learning_rate=0.001,\n        beta_1=0.9,\n        beta_2=0.999,\n        epsilon=1e-07,\n        amsgrad=False)\n\n        loss = tf.keras.losses.SparseCategoricalCrossentropy()\n\n        model.compile(loss = loss , optimizer = optimizer , metrics=['sparse_categorical_accuracy'])\n        \n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training Without Augmented Data As WarmUp**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(training_dataset , epochs = EPOCHS , validation_data = validation_dataset , steps_per_epoch = STEPS_PER_EPOCH , callbacks = [learning_rate_callback] )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Data Augmentation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"augmented_dataset = get_augmented_dataset()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training With Augmented Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(augmented_dataset , epochs = EPOCHS , validation_data = validation_dataset , steps_per_epoch = STEPS_PER_EPOCH , callbacks = [learning_rate_callback] )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Loading Testing Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_test_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Predictions**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Predicting\")\ntest_images_ds = test_dataset.map(lambda image , idnum : image)\nprob = model.predict(test_images_ds)\npred = np.argmax(prob , axis = -1)\nprint(pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Generating CSV For Submission**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Generating Csv\")\ntest_ids_ds = test_dataset.map(lambda image , idnum : idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\nnp.savetxt('submission.csv' , np.rec.fromarrays([test_ids , pred]), fmt=['%s' , '%d'] , delimiter=',',header='id,label' , comments='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}