{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from datetime import datetime\nstart_time=datetime.now()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TPU DETECTION**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**HYPERPARAMS**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16*strategy.num_replicas_in_sync\n\nIMAGE_SIZE = [1024, 1024]\n\nEPOCHS = 15\n\n\n#train tfrec files\nTRAINING_FILENAMES=tf.io.gfile.glob(GCS_DS_PATH+'/tfrecords/train*')\n\n#Splitting the training data\nTRAINING_FILENAMES,VALIDATION_FILENAMES=train_test_split(TRAINING_FILENAMES,test_size=0.2,random_state=7)\n\n#test tfrec files\nTEST_FILENAMES=tf.io.gfile.glob(GCS_DS_PATH+ '/tfrecords/test*.tfrec')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(TRAINING_FILENAMES)  , len(VALIDATION_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# *Preprocessing*","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.bfloat16) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image,0,3)\n    #image = tf.image.random_crop(image ,[224,224,3])\n    return image, label  \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True)\n    #dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    #dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    #dataset = dataset.shuffle(128)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES= count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS= NUM_VALIDATION_IMAGES // BATCH_SIZE\nprint('Dataset: {} training images,{} unlabeled test images'.format(NUM_TRAINING_IMAGES,NUM_TEST_IMAGES))\nprint(\"STEPS_PER_EPOCH are {}\".format(STEPS_PER_EPOCH))\nprint(\"VALIDATION_STEPS are {}\".format(VALIDATION_STEPS))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training Dataset**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#training preprocessed dataset \ntrain_ds=get_training_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds.element_spec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds=get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds.element_spec","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model imports**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet.tfkeras import EfficientNetB7\nfrom tensorflow.keras import *\nfrom tensorflow.keras.layers import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Callbacks**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.000001\nLR_MAX = 0.00005 * 16 #SETTING THE TPU CORES FOR EVERY DEVICE TO WORK ON\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 2\nLR_SUSTAIN_EPOCHS = 4\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\nes=tf.keras.callbacks.EarlyStopping(monitor='val_loss',mode='min',patience=4,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Training**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model():\n    base_model=EfficientNetB7(include_top=False,input_shape=(*IMAGE_SIZE,3),weights='imagenet')\n    base_model.trainable=False\n    inputs=Input(shape=(*IMAGE_SIZE,3))\n    X=base_model(inputs,training=False)\n    X=GlobalAveragePooling2D()(X)\n    X=Dense(1024,activation='relu')(X)\n    X=BatchNormalization()(X)\n    X=Dropout(0.3)(X)\n    outputs=Dense(1,activation='sigmoid')(X)\n    return Model(inputs,outputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model=create_model()\n    \n    model.compile(optimizer='rmsprop',loss='binary_crossentropy',metrics=[tf.keras.metrics.AUC()])\n    \n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_ds,\n          epochs=EPOCHS,\n          steps_per_epoch=STEPS_PER_EPOCH,\n          validation_data=valid_ds,\n          validation_steps=VALIDATION_STEPS,\n          callbacks=[es,lr_callback]\n         )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(\n    model.history.history['loss'],\n    model.history.history['val_loss'],\n    'loss',211\n)\ndisplay_training_curves(\n    model.history.history['auc'],\n    model.history.history['val_auc'],\n    'auc',212\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing dataset\ntest_ds=get_test_dataset(ordered=True)\n\ntest_images_ids=test_ds.map(lambda image ,idnum : image) #Retriving the image data from the test dataset\n\npredictions = model.predict(test_images_ids) #predictions on images\n\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch() #Retriving the Image name i.e., IDNUM\n\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') #Getting the image names for the csv file\n\nprediction_df=pd.DataFrame({'image_name':test_ids ,'target':np.concatenate(predictions)})\n\nprediction_df.to_csv('submission.csv',index=False) #generating the submission.csv file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Time taken\")\nprint(datetime.now()-start_time)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}