{"cells":[{"metadata":{"trusted":true},"cell_type":"markdown","source":"# import  efficientnet  to wok offline"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/efficientnet-keras-dataset/efficientnet_kaggle')\n! pip install -e ../input/efficientnet-keras-dataset/efficientnet_kaggle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom keras.preprocessing.image import ImageDataGenerator\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\n\nfrom tensorflow.keras import layers\n\n\n\nimport os\nimport tempfile\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nimport sklearn\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport efficientnet.tfkeras\nfrom tensorflow.keras.models import load_model\nimport math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nimport time\nimport json\nimport copy\nimport gc\nimport os\nimport time\nimport random\nfrom datetime import datetime\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.keras as efn \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# tpu detaction and config\n\nref  https://www.tensorflow.org/guide/tpu   https://www.kaggle.com/docs/tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"using gcs_path"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH =  KaggleDatasets().get_gcs_path('cassava-leaf-disease-classification')\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512 , 512]\nCLASSES = ['0', '1', '2', '3', '4']\nEPOCHS = 25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# working with tfrecoreds"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec'),\n    test_size=0.25, random_state=5\n)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/ld_test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# doing random data augmention on tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n    \n    dc = random.choice([\"flip\",\"rotat\" , \"contrast\" , \"brightnes\"])\n    if dc =='flip':\n        image = tf.image.random_flip_left_right(image)\n    elif dc =='rotat':\n        image = tf.image.random_flip_up_down(image)   \n    elif dc =='contrast':\n        image = tf.image.random_contrast(image, 0.2, 0.5)\n    elif dc =='brightnes':\n        image = tf.image.random_brightness(image , 0.2)\n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE) \n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# learning rate schudler"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler = keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate=1e-4, \n    decay_steps=10000, \n    decay_rate=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"layers = tf.keras.layers\nmodels = tf.keras.models\nlosses = tf.keras.losses\noptimizers = tf.keras.optimizers \nmetrics = tf.keras.metrics\nutils = tf.keras.utils\ncallbacks = tf.keras.callbacks","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# model creation function"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef create_model(met,trin=0.35):\n\n    base =  efn.EfficientNetB7(\n        include_top=False,\n        weights=\"imagenet\",\n        input_tensor=None,\n        input_shape=None,\n       \n    )\n    trin = round(len(base.layers)*trin)\n    for layer in base.layers[:-trin]:\n        layer.trainable = False\n    model = models.Sequential()\n    '''auglayer = tf.keras.Sequential([layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n                       layers.experimental.preprocessing.RandomRotation(0.2),\n                        layers.experimental.preprocessing.RandomContrast(1),\n                        layers.experimental.preprocessing.RandomZoom(0.2)])\n    model.add(keras.Input(shape=(512,512,3,)))\n    model.add(auglayer) \n    '''\n    model.add(base)\n    model.add(tf.keras.layers.GlobalAveragePooling2D())\n    #model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(1024, activation='relu'))\n    model.add(layers.Dropout(0.4))\n\n\n    model.add(layers.Dense(512, activation='relu'))\n\n    model.add(layers.Dropout(0.2))\n\n    model.add(layers.Dense(128, activation='relu'))\n\n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(5, activation=\"softmax\"))\n\n    #adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n    model.compile( optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler, beta_1=0.9, beta_2=0.999, epsilon=1e-08),\n        loss='sparse_categorical_crossentropy',  \n        metrics=met)\n    return model\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# defin the model within tpu stratgy and set parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    METRICS = [ \n          keras.metrics.SparseCategoricalAccuracy(name='accuracy')\n    ]\n    model = create_model(met=METRICS,trin=0.5)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n#checkpoits for saving best wights based on val_loss \nmcp_save = callbacks.ModelCheckpoint('mamon-checkpoint-tpu-efnb7-5hlf.hdf5', save_best_only=True, monitor='val_loss', mode='min')\n\n#start training while history will contain metrices\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS,\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS,callbacks=[mcp_save])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print out variables available to us  \nprint(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create learning curves to evaluate model performance\nhistory_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['accuracy', 'val_accuracy']].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Evaluate on test data\")\nresults = model.evaluate(valid_dataset)\nprint(\"test loss, test acc:\", results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.models.load_model('./mamon-checkpoint-tpu-efnb7-5hlf.hdf5')\nprint(\"Evaluate on test data\")\nresults = model.evaluate(valid_dataset)\nprint(\"test loss, test acc:\", results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('efnetb7-acc88.hdf5')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}