{"cells":[{"metadata":{},"cell_type":"markdown","source":"<h2>Stratified GroupKFold with EFN & TFRecords</h2>\n\nThis is a sample notebook for Ranzr clip competition which presents Stratified GroupKFold cross validation and Efficient Net architecture getting trained using TPUs. This notebook allows you to configure your model training by letting you choose efn architechutre, TFRecords with different image shapes and many more parameters.\n\nAll the Dataset used in this notebook is public and can be found at : [ [(128x128)](https://www.kaggle.com/prateek0x/ranzcr-128x128) , [(256x256)](https://www.kaggle.com/prateek0x/ranzcr-256x256) , [(384x384)](https://www.kaggle.com/prateek0x/ranzcr-384x384) , [(512x512)](https://www.kaggle.com/prateek0x/ranzcr-512x512)  ]\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np \nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nimport warnings, gc, random, math, os, re\nfrom sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE= \"TPU\"\n\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        print(\"Please select TPU as accelrator.\")\n\n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 5\nLEARNING_RATE = 1e-5\nBATCH_SIZE = 32 * REPLICAS\nEPOCHS = 10\nHEIGHT = 512 # 128, 256, 384\nWIDTH = 512 # 128, 256, 384\nRS_HEIGHT = 512 \nRS_WIDTH= 512\nEFN = 4\nN_CLASSES =11\nES_PATIENCE = 4\nCHANNELS =3\nDISPLAY_PLOT =True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nGCS_PATH = KaggleDatasets().get_gcs_path('ranzcr-{0}x{1}'.format(HEIGHT,WIDTH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    # Shear\n    if p_shear > .2:\n        if p_shear > .6:\n            image = transform_shear(image, HEIGHT, shear=20.)\n        else:\n            image = transform_shear(image, HEIGHT, shear=-20.)\n            \n    # Rotation\n    if p_rotation > .2:\n        if p_rotation > .6:\n            image = transform_rotation(image, HEIGHT, rotation=45.)\n        else:\n            image = transform_rotation(image, HEIGHT, rotation=-45.)\n    \n    \n    # Flips\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    if p_spatial > .75:\n        image = tf.image.transpose(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n        \n    # Pixel-level transforms\n    if p_pixel_1 >= .4:\n        image = tf.image.random_saturation(image, lower=.7, upper=1.3)\n    if p_pixel_2 >= .4:\n        image = tf.image.random_contrast(image, lower=.8, upper=1.2)\n    if p_pixel_3 >= .4:\n        image = tf.image.random_brightness(image, max_delta=.1)\n        \n    # Crops\n    if p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.6)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.8)\n    elif p_crop > .4:\n        crop_size = tf.random.uniform([], int(HEIGHT*.6), HEIGHT, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n            \n    image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU utility Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\ndef decode_image(image_data):\n    \"\"\"\n        1. Decode a JPEG-encoded image to a uint8 tensor.\n        2. Cast tensor to float and normalizes (range between 0 and 1).\n        3. Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled=True):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n        3. If 'labeled' returns (image, label) if not (image, name).\n    \"\"\"\n    if labeled:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'ETT - Abnormal' : tf.io.FixedLenFeature([], tf.int64),\n            'ETT - Borderline' : tf.io.FixedLenFeature([], tf.int64),\n            'ETT - Normal' : tf.io.FixedLenFeature([], tf.int64),\n            \"NGT - Abnormal\" : tf.io.FixedLenFeature([], tf.int64),\n            'NGT - Borderline' : tf.io.FixedLenFeature([], tf.int64),\n            'NGT - Incompletely Imaged' : tf.io.FixedLenFeature([], tf.int64),\n            'NGT - Normal' : tf.io.FixedLenFeature([], tf.int64),\n            'CVC - Abnormal' : tf.io.FixedLenFeature([], tf.int64),\n            'CVC - Borderline': tf.io.FixedLenFeature([], tf.int64),\n            'CVC - Normal': tf.io.FixedLenFeature([], tf.int64),\n            'StudyInstanceUID' : tf.io.FixedLenFeature([], tf.string),\n            'Swan Ganz Catheter Present' : tf.io.FixedLenFeature([], tf.int64),\n            \"StudyInstanceUID\":  tf.io.FixedLenFeature([], tf.string)\n        }\n    else:\n        TFREC_FORMAT = {\n            'image': tf.io.FixedLenFeature([], tf.string), \n            'StudyInstanceUID': tf.io.FixedLenFeature([], tf.string), \n        }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    \n    if labeled:\n        #tf.cast(example['target'], tf.int32),\n        \n        label_or_name = tf.stack([\n            tf.cast( example['ETT - Abnormal'],tf.int64),\n            tf.cast( example['ETT - Borderline'],tf.int64),\n            tf.cast( example['ETT - Normal'],tf.int64),\n            tf.cast( example[\"NGT - Abnormal\"],tf.int64),\n            tf.cast( example['NGT - Borderline'],tf.int64),\n            tf.cast( example['NGT - Incompletely Imaged'],tf.int64),\n            tf.cast( example['NGT - Normal'],tf.int64),\n            tf.cast( example['CVC - Abnormal'],tf.int64),\n            tf.cast( example['CVC - Borderline'],tf.int64),\n            tf.cast( example['CVC - Normal'],tf.int64),\n            tf.cast( example['Swan Ganz Catheter Present'],tf.int64)\n            ])\n        \n    else:\n        label_or_name =  example['StudyInstanceUID']\n        \n    return image, label_or_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    \"\"\"\n        Create a Tensorflow dataset from TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef get_dataset(FILENAMES, labeled=True, ordered=False, repeated=False, augment=False):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(FILENAMES, labeled=labeled, ordered=ordered)\n    if augment:\n        dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if repeated:\n        dataset = dataset.repeat()\n    if not ordered:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 1e-5\nLR_MIN = 1e-5\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nN_CYCLES = .5 \n\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        progress = (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)\n        lr = LR_MAX * (0.5 * (1.0 + tf.math.cos(math.pi * N_CYCLES * 2.0 * progress)))\n        if LR_MIN is not None:\n            lr = tf.math.maximum(LR_MIN, lr)\n            \n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef model_fn(dim=HEIGHT):\n    inp = tf.keras.layers.Input(shape=(dim,dim,3))\n    base = EFNS[EFN](input_shape=(dim,dim,3),include_top=False,weights='noisy-student')\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(N_CLASSES,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(lr=LEARNING_RATE) \n    loss = \"binary_crossentropy\"\n    model.compile(optimizer=opt,loss=loss,metrics=[tf.keras.metrics.AUC(multi_label=True)])\n    \n    return model\n\nm = model_fn()\nm.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS, shuffle=True, random_state=seed)\noof_pred = []; oof_labels = []; history_list = []\n\nfor fold,(idxT, idxV) in enumerate(skf.split(np.arange(15))):\n    if tpu:\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    print(f'\\nFOLD: {fold+1}')\n    print(f'TRAIN: {idxT} VALID: {idxV}')\n\n    # Create train and validation sets\n    # Create train and validation sets\n    TRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxT])    \n    VALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/Id_train%.2i*.tfrec' % x for x in idxV])\n    \n    np.random.shuffle(TRAIN_FILENAMES)\n    \n    ct_train = count_data_items(TRAIN_FILENAMES)\n    ct_valid = count_data_items(VALID_FILENAMES)\n    \n    ## MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = model_fn()\n        \n    model_path = f'model_{fold}.h5'\n    es = EarlyStopping(monitor='val_auc', mode='max', \n                       patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n\n    ## TRAIN\n    history = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True, augment=True), \n                        validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False), \n                        steps_per_epoch=(ct_train // BATCH_SIZE), \n                        callbacks=[es, LearningRateScheduler(lrfn, verbose=1)], \n                        epochs=EPOCHS,  \n                        #callbacks=[es], \n                        verbose=1).history\n      \n    history_list.append(history)\n    # Save last model weights\n    model.save_weights(model_path)\n\n    # OOF predictions\n    ds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False, augment=False)\n    oof_labels.append([target.numpy() for img, target in iter(ds_valid.unbatch())])\n    x_oof = ds_valid.map(lambda image, image_name: image)\n    oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n    \n    \n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(len(history['auc'])),history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(len(history['auc'])),history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history['val_auc'] )\n        y = np.max( history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4')\n        plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        \n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(len(history['loss'])),history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(len(history['loss'])),history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history['val_loss'] )\n        y = np.min( history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728')\n        plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('Fold %i,Image Size %i, EfficientNet B%i, batch_size %i '% \n                (fold+1,HEIGHT,EFN,BATCH_SIZE),size=18)\n        plt.legend(loc=3)\n        plt.show()  \n        \n    \n    ## RESULTS\n    print(f\"#### FOLD {fold+1} OOF Accuracy = {np.max(history['val_auc']):.3f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1>!The End</h1>Updating...."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}