{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is Mark Lavin's versions of the excellent notebook by Wojtek Rosa https://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-train\n\nLavin-specific parts are enabled by setting \"LAVIN\" to True","metadata":{}},{"cell_type":"code","source":"LAVIN = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is HuBMAP TF keras training with TPU notebook - full inference and submission using GPU notebook can be found here:\nhttps://www.kaggle.com/wrrosa/hubmap-tf-with-tpu-efficientunet-512x512-subm\n\n# Versions\n* V1-V6 init\n* V7: 4-CV efficientunetb0 512x512 (LB .834)\n* V8: loss bce, fixed dice_coe function for tpu (LB .835)\n* V9: efficientunetb1, added oof metrics.json (CV .871, LB .830)\n* V10: efficientunetb4 (CV .874, LB .839) \n* V11: \n    * updated files paths (moved files to train folder)\n    * efficientunetb7 (memory issue on 3-rd fold)\n* V12: P['BATCH_COE'] = 4, efficientunetb7 (CV .858, LB .835)\n* V13: efficientunetb4, P['BATCH_COE'] = 8, P['SEED'] = 1 (just rerun nb V10 with best lb and consume tpu quota ;) ) (CV .877, LB 0.836)\n* V14: efficientunetb4, add overlapped tiles and P['EPOCHS'] = 30 and P['SEED'] = 0 (...)\n\n","metadata":{}},{"cell_type":"markdown","source":"# Refferences:\n* @marcosnovaes  https://www.kaggle.com/marcosnovaes/hubmap-looking-at-tfrecords and https://www.kaggle.com/marcosnovaes/hubmap-unet-keras-model-fit-with-tpu\n* @mgornergoogle https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n* qubvel https://github.com/qubvel/segmentation_models  !! 25 available backbones for each of 4 architectures\n* @kool777, @joshi98kishan https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n* @cdeotte https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n","metadata":{}},{"cell_type":"markdown","source":"# Init - parameters, packages, gcs_paths, tpu","metadata":{}},{"cell_type":"code","source":"! pip install segmentation_models -q\n%matplotlib inline\n\nimport os\nif LAVIN:\n    import sys\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport segmentation_models as sm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\n\nfrom kaggle_secrets import UserSecretsClient\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\n\nos.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"P = {}\nP['EPOCHS'] = 30\nP['BACKBONE'] = 'efficientnetb4' \nP['NFOLDS'] = 5 # Divide 15 train images into five \"folds\" for training\nP['SEED'] = 0\nP['VERBOSE'] = 0\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 4 # ### 8 # BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\nP['DIM'] = 512 # 256 128\nP['LR'] = 5e-4 \nif LAVIN:\n    P['OVERLAPP'] = False\n    P['PIXEL_OVERLAP'] = 64\n    P['NORMALIZE_IMAGE'] = False\n    P['DO_AUGMENT'] = False\nelse:\n    P['OVERLAPP'] = True\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GCS_PATHS","metadata":{}},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)\n\nif LAVIN:\n    # ### GCS_PATH = KaggleDatasets().get_gcs_path(f'overlapping-tiles-from-normalized-train-images')\n    # ### GCS_PATH = KaggleDatasets().get_gcs_path(f'hubmap-train-overlapping-tiled-images')\n    GCS_PATH = KaggleDatasets().get_gcs_path( \"new-512x512x64-train-no-norm-no-aug-skip-blank\" )\n    # ###                                                     GCS_PATH = KaggleDatasets().get_gcs_path( \"new-512x512x64-train-normalize-no-aug-skip-blank\" )\nelse:\n    GCS_PATH = KaggleDatasets().get_gcs_path(f'hubmap-tfrecords-1024-{P[\"DIM\"]}')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')\nALL_TRAINING_FILENAMES","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if P['OVERLAPP']:\n    GCS_PATH2 = KaggleDatasets().get_gcs_path(f'hubmap-tfrecords-1024-{P[\"DIM\"]}-2')\n    ALL_TRAINING_FILENAMES2 = tf.io.gfile.glob(GCS_PATH2 + '/train2/*.tfrec')\n    #print(ALL_TRAINING_FILENAMES2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"[-_]([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint('NUM_TRAINING_IMAGES:' )\nif P['OVERLAPP']:\n    print(count_data_items(ALL_TRAINING_FILENAMES2)+count_data_items(ALL_TRAINING_FILENAMES))\nelse:\n    print(count_data_items(ALL_TRAINING_FILENAMES))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets pipeline","metadata":{}},{"cell_type":"code","source":"if LAVIN:\n    DIM = P[ 'DIM' ] + 2 * P[ 'PIXEL_OVERLAP' ]\nelse:\n    DIM = P['DIM']\n\ndef _parse_image_function(example_proto,augment = True):\n    if LAVIN:\n        image_feature_description = {\n            'img_index': tf.io.FixedLenFeature([], tf.int64 ),\n            'height': tf.io.FixedLenFeature([], tf.int64 ),  # NOTE this and \"width\" >includes< 2 * pixel_overlap\n            'width': tf.io.FixedLenFeature([], tf.int64 ),\n            'pixel_overlap': tf.io.FixedLenFeature([], tf.int64 ),\n            'num_channels': tf.io.FixedLenFeature([], tf.int64 ),\n            'image': tf.io.FixedLenFeature([], tf.string ),\n            'mask' : tf.io.FixedLenFeature([], tf.string ),\n            'tile_id':  tf.io.FixedLenFeature([], tf.int64 ),\n            'tile_col_pos': tf.io.FixedLenFeature([], tf.int64 ),\n            'tile_row_pos': tf.io.FixedLenFeature([], tf.int64 ),\n        }\n    else:\n        image_feature_description = {\n            'image': tf.io.FixedLenFeature([], tf.string),\n            'mask': tf.io.FixedLenFeature([], tf.string)\n        }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    if LAVIN:\n        image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    else:\n        image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))\n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    if LAVIN:\n        dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO, compression_type = \"GZIP\" )\n    else:\n        dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    if LAVIN:\n        dataset = load_dataset(TRAINING_FILENAMES, augment = P['DO_AUGMENT'] )\n    else:\n        dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered, augment = False)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(output, target, axis = None, smooth=1e-10):\n    if LAVIN:\n        # Trim off the overlap from each side of the \"output\" and \"target\" tiles\n        # before comparing them:\n        overlap = P[ 'PIXEL_OVERLAP' ]\n        output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 ) [ :, overlap : - overlap , overlap : - overlap ]\n        target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 ) [ :, overlap : - overlap , overlap : - overlap ]\n    else:\n        output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n        target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = tf.reshape( y_true, [ -1 ] )  # ### K.flatten(y_true)\n    y_pred_pos = tf.reshape( y_pred, [ -1 ] )  # ### K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    result = (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\n    return result\n\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})\n\nif LAVIN:\n    def windowed_focal_tversky_loss( y_true, y_pred ):\n        overlap = P[ 'PIXEL_OVERLAP' ]\n        windowed_y_true = y_true[ :, overlap : - overlap, overlap : - overlap ]\n        windowed_y_pred = y_pred[ :, overlap : - overlap, overlap : - overlap ]\n        loss = focal_tversky_loss( windowed_y_true, windowed_y_pred )\n        del windowed_y_true\n        del windowed_y_pred\n        return loss\n    get_custom_objects().update({\"windowed_focal_tversky\": windowed_focal_tversky_loss})\n    \n    def windowed_dice_coe_loss( output, target ):\n        return 1 - dice_coe( output, target )\n    \n    get_custom_objects().update( { \"windowed_dice_coe_loss\" : windowed_dice_coe_loss } )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model fit","metadata":{}},{"cell_type":"code","source":"M = {}\nmetrics = ['loss','dice_coe','accuracy']\nfor fm in metrics:\n    M['val_'+fm] = []\n\nfold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n    \n    print('#'*35); print('############ FOLD ',fold+1,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    if P['OVERLAPP']:\n        TRAINING_FILENAMES += [ALL_TRAINING_FILENAMES2[fi] for fi in tr_idx]\n    \n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():   \n        model = sm.Unet(P['BACKBONE'], encoder_weights='imagenet')\n        model.compile(optimizer = tf.keras.optimizers.Adam(lr = P['LR']),\n                      loss = \"windowed_focal_tversky\", # ### tf.keras.losses.BinaryCrossentropy(),#'focal_tversky',\n                      metrics=[dice_coe,'accuracy'])\n        \n    # CALLBACKS\n    checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True)\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coe',mode = 'max', patience=10, restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.00001)\n        \n    print(f'Training Model Fold {fold+1}...')\n    history = model.fit(\n        get_training_dataset(),\n        epochs = P['EPOCHS'],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        callbacks = [checkpoint, reduce,early_stop],\n        validation_data = get_validation_dataset(),\n        verbose=P['VERBOSE']\n    )   \n    \n    # SAVE METRICS\n    m = model.evaluate(get_validation_dataset(),return_dict=True)\n    for fm in metrics:\n        M['val_'+fm].append(m[fm])\n    \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if P['DISPLAY_PLOT']:        \n        plt.figure(figsize=(15,5))\n        n_e = np.arange(len(history.history['dice_coe']))\n        plt.plot(n_e,history.history['dice_coe'],'-o',label='Train dice_coe',color='#ff7f0e')\n        plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')\n        x = np.argmax( history.history['val_dice_coe'] ); y = np.max( history.history['val_dice_coe'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max dice_coe\\n%.2f'%y,size=14)\n        plt.ylabel('dice_coe',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(n_e,history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(n_e,history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### WRITE METRICS\nimport json\nfrom datetime import datetime\nM['datetime'] = str(datetime.now())\nfor fm in metrics:\n    M['oof_'+fm] = np.mean(M['val_'+fm])\n    print('OOF '+ fm + ' '+ str(M['oof_'+fm]))\nwith open('metrics.json', 'w') as outfile:\n    json.dump(M, outfile)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}