{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Introduction\n\nI made this kernel to burn my TPU. Let's burn the TPU together. :)\n- For train 1 fold, it need about 2h 30m.\n\n\n(Currently, only image data is used.)\n- It is base for this competition using tf.keras.\n- It is created to guide beginners and starters.\n\n\n> Using this :\n>> bfloat16\n>\n>> Focal Loss\n>\n>> Stratified GroupKFold Splitting\n\n\nThanks for reading it! :)\n\nIf you like this kernel, Please Upvote for me to be motivated!\n\n\n> References\n* https://www.kaggle.com/shonenkov/merge-external-data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\n!pip install tensorflow-addons==0.9.1    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport time\nimport random\nimport warnings\n\nimport keras\nimport tensorflow as tf\n\nfrom tensorflow.keras.models import Sequential , Model\nfrom tensorflow.keras.layers import Dense, Conv2D, ReLU, Dropout, Flatten, Activation, GlobalAveragePooling2D, AveragePooling2D\nfrom tensorflow.keras.layers import BatchNormalization\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import DenseNet121\nimport tensorflow.keras.applications.inception_resnet_v2 as InceptionResnetV2\nimport tensorflow.keras.applications.inception_resnet_v2 as InceptionResnetV2\n\nimport os\nimport tensorflow as tf, tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\nfrom tensorflow.keras.utils import plot_model\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\nprint(tf.__version__)\nprint(tf.keras.__version__)\n\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['TF_KERAS'] = '1'\n\nseed = 2020\nseed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data\n### Merge External Data\n> https://www.kaggle.com/shonenkov/merge-external-data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('melanoma-merged-external-data-512x512-jpeg')\nGCS_TRAIN_PATH = '/512x512-dataset-melanoma/512x512-dataset-melanoma/'\nGCS_TEST_PATH = '/512x512-test/512x512-test/'\n\npath='../input/siim-isic-melanoma-classification/'\n#path='../input/melanoma-merged-external-data-512x512-jpeg/'\n\n#train_df = pd.read_csv(path + 'train.csv')\ntest_df = pd.read_csv(path + 'test.csv')\nsub = pd.read_csv(path + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds_08062020.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Callback","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mc = ModelCheckpoint('best_model.h5',monitor='val_loss',mode='min',save_best_only=True,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor=tf.keras.metrics.AUC(),mode='max',verbose=1,patience=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Checkpoint","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_checkpoint(name='best_model'):\n    model_checkpoint = ModelCheckpoint(name + '.h5', monitor=tf.keras.metrics.AUC(), \n                               mode='min', verbose=1,\n                               save_best_only=True)\n    \n    return model_checkpoint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### With TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry :\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU :',tpu.master())\nexcept ValueError : \n    tpu = None\n    \nif tpu :\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse :\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"Replicas :\", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8 * strategy.num_replicas_in_sync\nEPOCHS = 2 # you can change epochs. e.g. 40\nimg_size = 512\nnb_classes = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset and transform","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None, image_size=(img_size,img_size)) :\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.bfloat16) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label == None :\n        return image\n    else :\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bool_random_brightness = False\nbool_random_contrast = False\nbool_random_hue = False\nbool_random_saturation = False\n\ndef data_augment(image, label=None):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if bool_random_brightness:\n        image = tf.image.random_brightness(image,0.2,seed=seed)\n    if bool_random_contrast:\n        image = tf.image.random_contrast(image,0.6,1.4, seed=seed)\n    if bool_random_hue:\n        image = tf.image.random_hue(image,0.07,seed=seed)\n    if bool_random_saturation:\n        image = tf.image.random_saturation(image,0.5,1.5,seed=seed)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold_number = 0\n\ntrain_paths = train_df[train_df.fold != fold_number].image_id.apply(lambda x : GCS_DS_PATH + GCS_TRAIN_PATH + x + '.jpg').values\ntrain_labels = train_df[train_df.fold != fold_number].target\ntrain_labels = tf.cast(train_labels, tf.float32)\n\nvalid_paths = train_df[train_df.fold == fold_number].image_id.apply(lambda x : GCS_DS_PATH + GCS_TRAIN_PATH + x + '.jpg').values\nvalid_labels = train_df[train_df.fold == fold_number].target\nvalid_labels = tf.cast(valid_labels, tf.float32)\n\ntest_paths = sub.image_name.apply(lambda x : GCS_DS_PATH + GCS_TEST_PATH + x + '.jpg').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .cache()\n    .repeat()\n    .shuffle(1024)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset=(\n    tf.data.Dataset\n    .from_tensor_slices(test_paths)\n    .map(decode_image ,num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_paths, valid_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_test_data(test_paths,aug=False):\n    test_data = (\n        tf.data.Dataset.from_tensor_slices(test_paths)\n        .map(decode_image, num_parallel_calls = AUTO)\n        .map(data_augment, num_parallel_calls=AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n    )\n    \n    if aug == True :\n        test_data = test_data.map(data_augment ,num_parallel_calls = AUTO)\n    return test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nLR_START = 0.0001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.0001\nLR_RAMPUP_EPOCHS = 4\nLR_SUSTAIN_EPOCHS = 6\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_scheduler = LearningRateScheduler(lrfn , verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_addons as tfa\n\nbool_focal_loss = True\nbool_label_smoothing = False\n\ndef get_model(name):\n\n    if name == \"EfficientNet\":        \n        base_model = efn.EfficientNetB0(weights='noisy-student',\n                               include_top = False,\n                               input_shape=(img_size,img_size,3)\n                              )            \n    elif name == 'ResNet':\n        base_model = ResNet50(weights='imagenet',\n                              include_top=False,\n                              input_shape=(img_size,img_size,3))\n    elif name == 'DenseNet':       \n        base_model = DenseNet121(weights='imagenet',\n                                 include_top=False,\n                                 input_shape=(img_size,img_size,3))\n    elif name == 'MobileNet':\n        base_model = MobileNet(weights='imagenet', \n                               include_top=False,\n                               input_shape=(img_size,img_size,3))\n    elif name == 'IncepResnet':       \n        base_model = InceptionResNetV2(weights='imagenet',\n                                       include_top=False,\n                                       input_shape=(img_size,img_size,3))\n    elif name == 'CumstomModel':\n        base_model = construct_model() # it need to be modified\n          \n    x = base_model.output\n    x = GlobalAveragePooling2D()(x)\n    x = L.Dense(1024,activation='relu')(x)\n    x = L.Dropout(0.5)(x,training=True)\n    predictions = Dense(1 ,activation='sigmoid')(x)\n    \n    if bool_focal_loss : \n        my_loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    elif bool_label_smoothing :\n        my_loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1)\n    else :\n        my_loss = 'binary_crossentropy'\n        \n    model = Model(inputs=base_model.input, outputs=predictions)\n    model.compile(optimizer='adam', loss=my_loss, metrics=['accuracy', tf.keras.metrics.AUC()])  \n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EfficientNet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() : \n   # mc = get_model_checkpoint('EfficientNet')\n    model_effnet = get_model(\"EfficientNet\")\n\n    history = model_effnet.fit(\n        train_dataset, \n        epochs=EPOCHS, \n        callbacks=[lr_scheduler, mc],\n        steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,\n        validation_data=valid_dataset,\n        validation_steps=valid_labels.shape[0] // BATCH_SIZE\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_effnet = model_effnet.predict(test_dataset, verbose=1)\nsub.target = preds_effnet\nsub.to_csv('sub_effnet.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### With Test Time augmentation :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_num = 5\nprobabilities = []\nfor i in range(tta_num) :\n    print('TTA number :',i+1)\n    test_tta = create_test_data(test_paths)\n    prob = model_effnet.predict(test_tta)\n    probabilities.append(prob)\n    \n    \ntab = np.zeros((len(probabilities[1]),1))\nfor i in range(len(probabilities[1])) :\n    for j in range(tta_num) :\n        tab[i] += probabilities[j][i] \ntab = tab / tta_num\nsub['target'] = tab\nsub.to_csv('efficientNet_TTA.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}