{"cells":[{"metadata":{},"cell_type":"markdown","source":"###  Original notebook: https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm\n\nThanks to @Wojtek Rosa and his notebook using TPU with tf.keras.\nThe changes I made to the notebook:\n- pretrain for 4 epochs by freezing encoder layers and then train all layers at lower learning rate by using qubvel's segmentation_models set_trainable() function and encoder_freeze=True option.\n- the lower threshold=0.3 to determine prediction masks.\n- efficientnet b5 backbone with lower batch_size = 6*8\n\n    "},{"metadata":{},"cell_type":"markdown","source":"# Versions\n* V1-V6 init\n* V7: 4-CV efficientunetb0 512x512 (LB .834)\n* V8: loss bce, fixed dice_coe function for tpu (LB .835)\n* V9: efficientunetb1, added oof metrics.json (CV 0.871, LB 0.830)\n* V10: efficientunetb4"},{"metadata":{},"cell_type":"markdown","source":"# Credits:\n* @marcosnovaes  https://www.kaggle.com/marcosnovaes/hubmap-looking-at-tfrecords and https://www.kaggle.com/marcosnovaes/hubmap-unet-keras-model-fit-with-tpu\n* @mgornergoogle https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n* qubvel https://github.com/qubvel/segmentation_models  !! 25 available backbones for each of 4 architectures\n* https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n"},{"metadata":{},"cell_type":"markdown","source":"# Init - parameters, packages, gcs_paths, tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"debug = False # True False\nn_rows = 1000 if not debug else 2\nP = {}\nP['EPOCHS_PRETRAIN'] = 1 if not debug else 1\nP['EPOCHS'] = 58 if not debug else 1 # 60\nP['BACKBONE'] = 'efficientnetb5' \nP['NFOLDS'] = 4 if not debug else 2 # 4\nP['SEED'] = 2\nP['VERBOSE_STAGE2'] = 0\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 6  # 8 # BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\nP['DIM'] = 512 # 256 128\nP['LR_pretrain'] = 1e-3  #5e-4  # 3.5e-4 # 5e-4 \nP['LR_MULTIPLIER'] = 0.35\nP['ESTOP_PATIENCE'] = 12 # was 10\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)\n\nprint(P)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"! pip install segmentation_models -q\n%matplotlib inline\n\nimport os\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport segmentation_models as sm\nfrom segmentation_models.utils import set_trainable\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GCS_PATHS"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(f'../input/hubmap-tfrecords-1024-{P[\"DIM\"]}/train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(f'hubmap-tfrecords-1024-{P[\"DIM\"]}')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/train/*.tfrec'))[:n_rows]\nALL_TRAINING_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint(count_data_items(ALL_TRAINING_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"DIM = P['DIM']\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered, augment = False)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(output, target, axis = None, smooth=1e-10):\n    output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n    target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = K.flatten(y_true)\n    y_pred_pos = K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model fit"},{"metadata":{"trusted":true},"cell_type":"code","source":"M = {}\nmetrics = ['loss','dice_coe','accuracy']\nfor fm in metrics:\n    M['val_'+fm] = []\n\nfold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n    \n    print('#'*35); print('############ FOLD ',fold+1,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    \n    print(\"num training files {}, num validation files {}\".format(len(TRAINING_FILENAMES), len(VALIDATION_FILENAMES)))\n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():   \n        model = sm.Unet(P['BACKBONE'], encoder_weights='imagenet', encoder_freeze=True)\n        model.compile( optimizer = tf.keras.optimizers.Adam(lr = P['LR_pretrain']),\n                       loss = tf.keras.losses.BinaryCrossentropy(),\n                        metrics=[dice_coe,'accuracy']\n                      )\n\n        # PRETRAIN\n        print(\"\\n===== Pretraining ... =========\")\n        _ = model.fit(\n            get_training_dataset(),\n            epochs = P['EPOCHS_PRETRAIN'],\n            steps_per_epoch = STEPS_PER_EPOCH,\n            #validation_data = get_validation_dataset(),\n            verbose=2 # P['VERBOSE']\n            )   \n\n        \n        set_trainable(model, recompile=False) # set all layers trainable and recompile model\n        print(\"\\nAfter pretraining - model.optimizer.lr = \", model.optimizer.lr)\n        model.optimizer.lr = model.optimizer.lr * P['LR_MULTIPLIER']\n        print(\"\\nAdjusted model.optimizer.lr before main training = \", model.optimizer.lr)\n        model.compile(optimizer = model.optimizer,\n                      loss = tf.keras.losses.BinaryCrossentropy(), # model.loss,  # sm.BinaryCELoss()\n                      metrics=[dice_coe,'accuracy'])\n                      \n                    # tf.keras.optimizers.Adam(lr = P['LR']),\n                    #  loss = tf.keras.losses.BinaryCrossentropy(),#'focal_tversky',\n                    #  metrics=[dice_coe,'accuracy'])\n\n    \n    # CALLBACKS\n    checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model-fold-%i.h5'%fold,\n                                 verbose=0, monitor='val_dice_coe', patience = 10,\n                                 mode='max', save_best_only=True)\n    \n    # early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coe',mode = 'max', patience=P['ESTOP_PATIENCE'], restore_best_weights=True)\n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss',mode = 'min', patience=P['ESTOP_PATIENCE'], restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.00001)\n        \n    print('\\n ============ Training Model... ================')\n    history = model.fit(\n        get_training_dataset(),\n        epochs = P['EPOCHS'],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        callbacks = [checkpoint, reduce,early_stop],\n        validation_data = get_validation_dataset(),\n        verbose=P['VERBOSE_STAGE2']\n    )   \n    \n    # SAVE METRICS\n    m = model.evaluate(get_validation_dataset(),return_dict=True)\n    for fm in metrics:\n        M['val_'+fm].append(m[fm])\n    \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if P['DISPLAY_PLOT']:        \n        plt.figure(figsize=(15,5))\n        n_e = np.arange(len(history.history['dice_coe']))\n        plt.plot(n_e,history.history['dice_coe'],'-o',label='Train dice_coe',color='#ff7f0e')\n        plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')\n        x = np.argmax( history.history['val_dice_coe'] ); y = np.max( history.history['val_dice_coe'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max dice_coe\\n%.2f'%y,size=14)\n        plt.ylabel('dice_coe',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(n_e,history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(n_e,history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()\n    \n    del model, checkpoint, early_stop, reduce, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### WRITE METRICS\nimport json\nfrom datetime import datetime\nM['datetime'] = str(datetime.now())\nfor fm in metrics:\n    M['oof_'+fm] = np.mean(M['val_'+fm])\n    print('OOF '+ fm + ' '+ str(round(M['oof_'+fm], 6)))\nwith open('metrics.json', 'w') as outfile:\n    json.dump(M, outfile)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}