{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Credits:\n@marcosnovaes https://www.kaggle.com/marcosnovaes/hubmap-looking-at-tfrecords and https://www.kaggle.com/marcosnovaes/hubmap-unet-keras-model-fit-with-tpu\n@mgornergoogle https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\nqubvel https://github.com/qubvel/segmentation_models !! 25 available backbones for each of 4 architectures\nhttps://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu","metadata":{}},{"cell_type":"markdown","source":"Init - parameters, packages, gcs_paths, tpu","metadata":{}},{"cell_type":"code","source":"P = {}\nP['EPOCHS'] = 60\nP['BACKBONE'] = 'efficientnetb0' \nP['NFOLDS'] = 4\nP['SEED'] = 0\nP['VERBOSE'] = 1\nP['DISPLAY_PLOT'] = True \nP['BATCH_COE'] = 8 # BATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\nP['DIM'] = 512 # 256 128\nP['LR'] = 0.00025\n\nimport yaml\nwith open(r'params.yaml', 'w') as file:\n    yaml.dump(P, file)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:45:19.641384Z","iopub.execute_input":"2022-07-01T10:45:19.641761Z","iopub.status.idle":"2022-07-01T10:45:19.713772Z","shell.execute_reply.started":"2022-07-01T10:45:19.641659Z","shell.execute_reply":"2022-07-01T10:45:19.712863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install segmentation_models -q\n%matplotlib inline\n\nimport os\nos.environ['SM_FRAMEWORK'] = 'tf.keras'\nimport glob\nimport segmentation_models as sm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.utils import get_custom_objects\nimport cv2\nimport math\n\nfrom kaggle_datasets import KaggleDatasets\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:45:19.715803Z","iopub.execute_input":"2022-07-01T10:45:19.716238Z","iopub.status.idle":"2022-07-01T10:45:37.996823Z","shell.execute_reply.started":"2022-07-01T10:45:19.716193Z","shell.execute_reply":"2022-07-01T10:45:37.995901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError: # no TPU found, detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nBATCH_SIZE = P['BATCH_COE'] * strategy.num_replicas_in_sync\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\nprint(\"BATCH_SIZE: \", str(BATCH_SIZE))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:45:37.998564Z","iopub.execute_input":"2022-07-01T10:45:37.998889Z","iopub.status.idle":"2022-07-01T10:45:44.011185Z","shell.execute_reply.started":"2022-07-01T10:45:37.998847Z","shell.execute_reply":"2022-07-01T10:45:44.010152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:45:44.012892Z","iopub.execute_input":"2022-07-01T10:45:44.013209Z","iopub.status.idle":"2022-07-01T10:45:44.219848Z","shell.execute_reply.started":"2022-07-01T10:45:44.013168Z","shell.execute_reply":"2022-07-01T10:45:44.218918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('hubmaphpa-im512-reduce2')\nALL_TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nALL_TRAINING_FILENAMES[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:45:44.224564Z","iopub.execute_input":"2022-07-01T10:45:44.224865Z","iopub.status.idle":"2022-07-01T10:45:53.141548Z","shell.execute_reply.started":"2022-07-01T10:45:44.224832Z","shell.execute_reply":"2022-07-01T10:45:53.140594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\nprint(count_data_items(ALL_TRAINING_FILENAMES))\nfor f in ALL_TRAINING_FILENAMES:\n    print(f+\":\"+str(count_data_items([f])))","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:46:12.982613Z","iopub.execute_input":"2022-07-01T10:46:12.982946Z","iopub.status.idle":"2022-07-01T10:46:13.045041Z","shell.execute_reply.started":"2022-07-01T10:46:12.982912Z","shell.execute_reply":"2022-07-01T10:46:13.044229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Datasets pipeline","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef transform_shear(image, height, shear, mask=False):\n    \n    '''\n    shear augmentation on image\n    and mask.\n    --------------------------------\n    \n    Arguments:\n    image -- input image\n    mask -- input mask\n    \n    Return:\n    image -- augmented image \n    mask -- augmented mask\n    '''\n    \n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    if mask:\n        return tf.reshape(d, [DIM,DIM,1])\n    \n    return tf.reshape(d, [DIM,DIM,3])\n\ndef transform_shift(image, height, h_shift, w_shift, mask=False):\n    \n    '''\n    shift augmentation on image\n    and mask.\n    --------------------------------\n    \n    Arguments:\n    image -- input image\n    mask -- input mask\n    \n    Return:\n    image -- augmented image \n    mask -- augmented mask\n    '''\n    \n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = h_shift * tf.random.uniform([1],dtype='float32') \n    width_shift = w_shift * tf.random.uniform([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    if mask:\n        return tf.reshape(d, [DIM,DIM,1])\n    \n    return tf.reshape(d, [DIM,DIM,3])","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:46:18.906338Z","iopub.execute_input":"2022-07-01T10:46:18.906634Z","iopub.status.idle":"2022-07-01T10:46:18.931536Z","shell.execute_reply.started":"2022-07-01T10:46:18.906605Z","shell.execute_reply":"2022-07-01T10:46:18.929174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIM = P['DIM']\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'mask': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['mask'],out_type='bool'),(DIM,DIM,1))        \n    \n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        rot = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n        if rot > 0.25:\n            k_= int(rot/0.25)            \n            image = tf.image.rot90(image, k=k_)\n            mask = tf.image.rot90(mask, k=k_)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.6, 1.4)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.7, 1.3)\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_brightness(image, 0.7, 1.3)\n        \n        '''        \n        shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n        shift = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n        ## SHEAR \n        if shear >= .5:\n            image = transform_shear(image, height=P['DIM'], shear=20.)\n            mask = transform_shear(mask, height=P['DIM'], shear=20., mask=True)\n\n        ## SHIFT\n        if shift >= .5:\n            image = transform_shift(image, height=P['DIM'], h_shift=15., w_shift=15.)\n            mask = transform_shift(mask, height=P['DIM'], h_shift=15., w_shift=15., mask=True)\n        '''\n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(_parse_image_function, num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(128, seed = P['SEED'])\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=True):\n    dataset = load_dataset(VALIDATION_FILENAMES, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE,drop_remainder=True)\n    #dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:46:22.030712Z","iopub.execute_input":"2022-07-01T10:46:22.031010Z","iopub.status.idle":"2022-07-01T10:46:22.049883Z","shell.execute_reply.started":"2022-07-01T10:46:22.030982Z","shell.execute_reply":"2022-07-01T10:46:22.048917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model","metadata":{}},{"cell_type":"code","source":"# https://tensorlayer.readthedocs.io/en/latest/_modules/tensorlayer/cost.html#dice_coe\ndef dice_coe(output, target, axis = None, smooth=1e-10):\n    output = tf.dtypes.cast( tf.math.greater(output, 0.5), tf. float32 )\n    target = tf.dtypes.cast( tf.math.greater(target, 0.5), tf. float32 )\n    inse = tf.reduce_sum(output * target, axis=axis)\n    l = tf.reduce_sum(output, axis=axis)\n    r = tf.reduce_sum(target, axis=axis)\n\n    dice = (2. * inse + smooth) / (l + r + smooth)\n    dice = tf.reduce_mean(dice, name='dice_coe')\n    return dice\n\n# https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\ndef tversky(y_true, y_pred, alpha=0.7, beta=0.3, smooth=1):\n    y_true_pos = K.flatten(y_true)\n    y_pred_pos = K.flatten(y_pred)\n    true_pos = K.sum(y_true_pos * y_pred_pos)\n    false_neg = K.sum(y_true_pos * (1 - y_pred_pos))\n    false_pos = K.sum((1 - y_true_pos) * y_pred_pos)\n    return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth)\ndef tversky_loss(y_true, y_pred):\n    return 1 - tversky(y_true, y_pred)\ndef focal_tversky_loss(y_true, y_pred, gamma=0.75):\n    tv = tversky(y_true, y_pred)\n    return K.pow((1 - tv), gamma)\n\nget_custom_objects().update({\"focal_tversky\": focal_tversky_loss})","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:46:25.181687Z","iopub.execute_input":"2022-07-01T10:46:25.182000Z","iopub.status.idle":"2022-07-01T10:46:25.193133Z","shell.execute_reply.started":"2022-07-01T10:46:25.181970Z","shell.execute_reply":"2022-07-01T10:46:25.192090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model fit","metadata":{}},{"cell_type":"code","source":"fold = KFold(n_splits=P['NFOLDS'], shuffle=True, random_state=P['SEED'])\nfor fold,(tr_idx, val_idx) in enumerate(fold.split(ALL_TRAINING_FILENAMES)):\n    \n    print('#'*35); print('############ FOLD ',fold+1,' #############'); print('#'*35);\n    print(f'Image Size: {DIM}, Batch Size: {BATCH_SIZE}')\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    TRAINING_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in tr_idx]\n    VALIDATION_FILENAMES = [ALL_TRAINING_FILENAMES[fi] for fi in val_idx]\n    STEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n    print(TRAINING_FILENAMES)\n    print(VALIDATION_FILENAMES)    \n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():   \n        model = sm.Unet(P['BACKBONE'], encoder_weights='imagenet')\n        loss = sm.losses.DiceLoss()        \n        model.compile(optimizer = tf.keras.optimizers.Adam(lr = P['LR']),\n        #model.compile(optimizer = tf.keras.optimizers.SGD(learning_rate = 0.2),\n                      #loss = tf.keras.losses.BinaryCrossentropy(),#'focal_tversky',\n                      loss = loss,\n                      metrics=[dice_coe,'accuracy'])\n        \n    # CALLBACKS\n    checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/model-fold-%i.h5'%fold,\n                                 verbose=P['VERBOSE'],monitor='val_dice_coe',patience = 10,\n                                 mode='max',save_best_only=True)\n    \n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_dice_coe',mode = 'max', patience=15, restore_best_weights=True)\n    reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.00001)\n    #reduce = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, min_lr=0.045)\n        \n    print('Training Model...')\n    history = model.fit(\n        get_training_dataset(),\n        epochs = P['EPOCHS'],\n        steps_per_epoch = STEPS_PER_EPOCH,\n        callbacks = [checkpoint, reduce,early_stop],\n        validation_data = get_validation_dataset(),\n        verbose=P['VERBOSE']\n    )   \n    # PLOT TRAINING\n    # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n    if P['DISPLAY_PLOT']:        \n        plt.figure(figsize=(15,5))\n        n_e = np.arange(len(history.history['dice_coe']))\n        plt.plot(n_e,history.history['dice_coe'],'-o',label='Train dice_coe',color='#ff7f0e')\n        plt.plot(n_e,history.history['val_dice_coe'],'-o',label='Val dice_coe',color='#1f77b4')\n        x = np.argmax( history.history['val_dice_coe'] ); y = np.max( history.history['val_dice_coe'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max dice_coe\\n%.2f'%y,size=14)\n        plt.ylabel('dice_coe',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(n_e,history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(n_e,history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T10:46:27.537628Z","iopub.execute_input":"2022-07-01T10:46:27.538070Z","iopub.status.idle":"2022-07-01T11:47:26.169323Z","shell.execute_reply.started":"2022-07-01T10:46:27.538040Z","shell.execute_reply":"2022-07-01T11:47:26.168253Z"},"trusted":true},"execution_count":null,"outputs":[]}]}