{"cells":[{"metadata":{},"cell_type":"markdown","source":"   This notebook implement the model use pretrain network, easy to undertand and custom.\n\n    If it is useful or you want to fork it :) please vote for this kernel.\n    \n    I am very happy if get some idea or suggest from you.\n    \n    Good look!!!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Import Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import math, os\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, GlobalMaxPooling2D\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nfrom albumentations import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\ndef seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 2048\nseed_everything(seed)\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# Data access\nfrom kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n# GCS_DS_PATH = './'\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configuration\nIMAGE_SIZE = [256, 256]\nEPOCHS = 15\nSEED = 777\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nSPLIT_VALIDATION = True\nSPLIT_FOLD = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixed Precision and/or XLA\nThe following booleans can enable mixed precision and/or XLA on GPU/TPU. By default TPU already uses some mixed precision but we can add more. These allow the GPU/TPU memory to handle larger batch sizes and can speed up the training process. The Nvidia V100 GPU has special Tensor Cores which get utilized when mixed precision is enabled. Unfortunately Kaggle's Nvidia P100 GPU does not have Tensor Cores to receive speed up.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MIXED_PRECISION = False\nXLA_ACCELERATE = True\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n\ntrain_paths = train.image_name.apply(lambda x: GCS_DS_PATH + '/jpeg/train/' + x + '.jpg').values\ntest_paths = test.image_name.apply(lambda x: GCS_DS_PATH + '/jpeg/test/' + x + '.jpg').values\n\ntrain_labels = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef data_augment(image, label=None):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.7, 1.3)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    image = tf.image.random_brightness(image, 0.1)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef get_training_dataset(paths, labels, do_aug=True):\n    dataset = tf.data.Dataset.from_tensor_slices((paths, labels)).map(decode_image, num_parallel_calls=AUTO)\n    if do_aug:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.cache()\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(paths, labels, ordered=False):\n    dataset = tf.data.Dataset.from_tensor_slices((paths, labels)).map(decode_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(paths, ordered=False):\n    dataset = tf.data.Dataset.from_tensor_slices(paths).map(decode_image, num_parallel_calls=AUTO).batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Custom Callbacks","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.000005, lr_max=0.000020, \n               lr_min=0.000001, lr_rampup_epochs=5, \n               lr_sustain_epochs=0, lr_exp_decay=.8):\n    lr_max = lr_max * strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn\n\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(build_lrfn(), verbose=1)\ncallbacks = [lr_schedule]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble model, Train, Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(name='EfficientNetB3', weights=\"imagenet\"):\n    with strategy.scope():\n        if name == 'InceptionResNetV2':\n            model = tf.keras.Sequential([\n                InceptionResNetV2(\n                    input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n                    weights=weights,\n                    include_top=False\n                ),\n                GlobalMaxPooling2D(),\n                Dense(1, activation='sigmoid')\n            ])\n        else:\n            model = tf.keras.Sequential([\n                efn.EfficientNetB3(\n                    input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),\n                    weights=weights,\n                    include_top=False\n                ),\n                GlobalAveragePooling2D(),\n                Dense(1, activation='sigmoid')\n            ])\n\n        model.compile(optimizer='adam',\n                      loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n                      metrics=['accuracy', tf.keras.metrics.AUC(name='auc')])\n    return model\n\ndef train(model_obj, training_data, validation_data):\n    model_obj.summary()\n    model_obj.fit(training_data,\n                    epochs=EPOCHS,\n                    callbacks=callbacks,\n                    steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,\n                    validation_data=validation_data)\n\n    return model_obj","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if SPLIT_VALIDATION:\n    trn_paths, val_paths, trn_labels, val_labels = train_test_split(train_paths, train_labels, test_size=0.2, random_state=seed)\nelse:\n    trn_paths, val_paths = train_paths, train_labels\n\nmodels = [\n    'EfficientNetB3_noisy-student',\n    'EfficientNetB3'\n]\n\nprobs = []\nfor mid, model_name in enumerate(models):\n    if model_name == 'EfficientNetB3_noisy-student':\n        model = get_model(name='EfficientNetB3', weights='noisy-student')\n    else:\n        model = get_model(name=model_name)\n\n    model = train(\n        model,\n        get_training_dataset(trn_paths, trn_labels, do_aug=True),\n        get_validation_dataset(val_paths, val_labels) if SPLIT_VALIDATION else None\n    )\n    model.save_weights('model_' + str(mid) + '.h5')\n\n    prob = model.predict(get_test_dataset(test_paths), verbose=1)\n    sub['target'] = prob\n    sub.to_csv('submission' + str(mid) + '.csv', index=False)\n    probs.append(prob)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['target'] = np.mean(probs, 0)\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !kaggle competitions submit -c siim-isic-melanoma-classification -f submission.csv -m \"Message\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}