{"cells":[{"metadata":{"trusted":true},"cell_type":"markdown","source":"# import  efficientnet  to wok offline"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/efficientnet-keras-dataset/efficientnet_kaggle')\n! pip install -e ../input/efficientnet-keras-dataset/efficientnet_kaggle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom keras.preprocessing.image import ImageDataGenerator\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\n\nfrom tensorflow.keras import layers\n\n\n\nimport os\nimport tempfile\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nimport sklearn\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport efficientnet.tfkeras\nfrom tensorflow.keras.models import load_model\nimport math, re, os\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nimport time\nimport json\nimport copy\nimport gc\nimport os\nimport time\nimport random\nfrom datetime import datetime\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn import model_selection, metrics\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.keras as efn \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# tpu detaction and config\n\nref  https://www.tensorflow.org/guide/tpu   https://www.kaggle.com/docs/tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"using gcs_path"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH =  KaggleDatasets().get_gcs_path('ranzcr-clip-catheter-line-classification')\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = 600\nEPOCHS = 25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# working with tfrecoreds"},{"metadata":{"trusted":true},"cell_type":"code","source":"#just exploraing the recoreds to understand them schema\nraw_dataset = tf.data.TFRecordDataset(GCS_PATH+'/train_tfrecords/00-1881.tfrec')\n\nfor raw_record in raw_dataset.take(1):\n    example = tf.train.Example()\n    example.ParseFromString(raw_record.numpy())\n    print(example)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) \n    image = tf.image.resize(image, (IMAGE_SIZE , IMAGE_SIZE))\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n     \n      \"CVC - Abnormal\": tf.io.FixedLenFeature([], tf.int64), \n    \n    \"CVC - Borderline\": tf.io.FixedLenFeature([], tf.int64), \n    \"CVC - Normal\": tf.io.FixedLenFeature([], tf.int64), \n    \"ETT - Abnormal\": tf.io.FixedLenFeature([], tf.int64), \n    \"ETT - Borderline\": tf.io.FixedLenFeature([], tf.int64), \n    \"ETT - Normal\": tf.io.FixedLenFeature([], tf.int64), \n    \"NGT - Abnormal\": tf.io.FixedLenFeature([], tf.int64), \n    \"NGT - Borderline\": tf.io.FixedLenFeature([], tf.int64), \n    \n    \n        \"NGT - Incompletely Imaged\": tf.io.FixedLenFeature([], tf.int64), \n\n        \"NGT - Normal\": tf.io.FixedLenFeature([], tf.int64), \n\n        \"Swan Ganz Catheter Present\": tf.io.FixedLenFeature([], tf.int64), \n\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = [example['CVC - Abnormal'],example['CVC - Borderline'],example['CVC - Normal'],example['ETT - Abnormal'],example['ETT - Borderline'],example['ETT - Normal'],\n                    example['NGT - Abnormal'],example['NGT - Borderline'],example['NGT - Incompletely Imaged'],example['NGT - Normal'],example['Swan Ganz Catheter Present']]\n   \n        return image, label\n    idnum = example['image_name']\n    return image, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec'),\n    test_size=0.2, random_state=5\n)\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test_tfrecords/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# doing random data augmention on tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)   \n    image = tf.image.random_contrast(image, 0.2, 0.5)\n    image = tf.image.random_brightness(image , 0.2)\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augmentval(image, label):\n\n    image /= 255.0\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTOTUNE) \n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(1000)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered) \n    dataset = dataset.map(data_augmentval, num_parallel_calls=AUTOTUNE) \n    dataset = dataset.repeat()    \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n\nprint('Dataset: {} training images, {} validation images, {} (unlabeled) test images'.format(\n    NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# learning rate schudler"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lrfn(epoch, bs=BATCH_SIZE, epochs=EPOCHS):\n    # Config\n    LR_START = 1e-6\n    LR_MAX = 2e-4\n    LR_FINAL = 1e-6\n    LR_RAMPUP_EPOCHS = 4\n    LR_SUSTAIN_EPOCHS = 0\n    DECAY_EPOCHS = epochs  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n    LR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / DECAY_EPOCHS) * math.pi\n        decay_factor= (tf.math.cos(decay_factor).numpy() + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plots the learning rate schedule\ndef show_lr_schedule(bs=BATCH_SIZE, epochs=EPOCHS):\n    rng = [i for i in range(epochs)]\n    y = [lrfn(x, bs=bs, epochs=epochs) for x in rng]\n    x = np.arange(epochs)\n    x_axis_labels = list(map(str, np.arange(1, epochs+1)))\n    print('init lr {:.1e} to {:.1e} final {:.1e}'.format(y[0], max(y), y[-1]))\n    \n    plt.figure(figsize=(30, 10))\n    plt.xticks(x, x_axis_labels, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    plt.plot(rng, y)\n    plt.grid()\n    plt.show()\n    \nshow_lr_schedule()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"layers = tf.keras.layers\nmodels = tf.keras.models\nlosses = tf.keras.losses\noptimizers = tf.keras.optimizers \nmetrics = tf.keras.metrics\nutils = tf.keras.utils\ncallbacks = tf.keras.callbacks","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# model creation function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(met,trin=0.35):\n\n    base =  efn.EfficientNetB7(\n        include_top=False,\n        weights=\"imagenet\",\n        \n        input_shape= None , # (IMAGE_SIZE , IMAGE_SIZE, 3),\n       \n    )\n    trin = round(len(base.layers)*trin)\n    for layer in base.layers[:-trin]:\n        layer.trainable = False\n    model = models.Sequential()\n    '''auglayer = tf.keras.Sequential([layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n                       layers.experimental.preprocessing.RandomRotation(0.2),\n                        layers.experimental.preprocessing.RandomContrast(1),\n                        layers.experimental.preprocessing.RandomZoom(0.2)])\n    model.add(keras.Input(shape=(512,512,3,)))\n    model.add(auglayer) \n    '''\n    model.add(base)\n    model.add(tf.keras.layers.GlobalAveragePooling2D())\n    #model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(1024, activation='relu'))\n    model.add(layers.Dropout(0.4))\n\n\n    model.add(layers.Dense(512, activation='relu'))\n\n    model.add(layers.Dropout(0.2))\n\n    model.add(layers.Dense(128, activation='relu'))\n\n    #model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(11, activation='sigmoid'))\n\n    #adam = optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n    model.compile( optimizer=tf.keras.optimizers.Adam(),\n        loss='binary_crossentropy',  \n        metrics=met)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# defin the model within tpu stratgy and set parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    METRICS = [ keras.metrics.AUC(multi_label=True)\n           \n    ]\n    model = create_model(met=METRICS,trin=0.75)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in model.layers:\n    print(layer.get_output_at(0).get_shape().as_list())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch, epochs=EPOCHS), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE\n#checkpoits for saving best wights based on val_loss \nmcp_save = callbacks.ModelCheckpoint('mamoncheckp600tpu-efnnet-halffreez-schdl.hdf5', save_best_only=True, monitor='val_loss', mode='min')\n\n#start training while history will contain metrices\nhistory = model.fit(train_dataset, \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    epochs=EPOCHS,\n                    validation_data=valid_dataset,\n                    validation_steps=VALID_STEPS ,callbacks=[mcp_save , lr_callback] )# , class_weight=class_weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print out variables available to us  \nprint(history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create learning curves to evaluate model performance\nhistory_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['auc', 'val_auc']].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = keras.models.load_model('./mamoncheckp600tpu-efnnet-halffreez-schdl.hdf5')\nprint(\"Evaluate on test data\")\nresults = model2.evaluate(valid_dataset, steps=VALID_STEPS)\nprint(\"test loss, test acc:\", results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.save('efnetb7-acc9218.hdf5')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}