{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:40:54.760478Z","iopub.execute_input":"2021-08-16T04:40:54.760883Z","iopub.status.idle":"2021-08-16T04:41:03.754304Z","shell.execute_reply.started":"2021-08-16T04:40:54.760791Z","shell.execute_reply":"2021-08-16T04:41:03.753144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import efficientnet.tfkeras as efn","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:04.571305Z","iopub.execute_input":"2021-08-16T04:41:04.571658Z","iopub.status.idle":"2021-08-16T04:41:11.142810Z","shell.execute_reply.started":"2021-08-16T04:41:04.571625Z","shell.execute_reply":"2021-08-16T04:41:11.141963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np, random,os, shutil\nfrom glob import glob\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\n#import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nprint('tf:',tf.__version__)\nimport math","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:11.144165Z","iopub.execute_input":"2021-08-16T04:41:11.144635Z","iopub.status.idle":"2021-08-16T04:41:11.445111Z","shell.execute_reply.started":"2021-08-16T04:41:11.144585Z","shell.execute_reply":"2021-08-16T04:41:11.444103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE=\"TPU\"\nif DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:12.339840Z","iopub.execute_input":"2021-08-16T04:41:12.340183Z","iopub.status.idle":"2021-08-16T04:41:18.228453Z","shell.execute_reply.started":"2021-08-16T04:41:12.340154Z","shell.execute_reply":"2021-08-16T04:41:18.227458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('siimcovidtfrecords')\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH+'/train*.tfrec')))\nfiles_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH+'/test*.tfrec')))\nnum_train_files = len(files_train)\nnum_test_files  = len(files_test)\nprint('train_files:',num_train_files)\nprint('test_files:',num_test_files)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:18.229839Z","iopub.execute_input":"2021-08-16T04:41:18.230116Z","iopub.status.idle":"2021-08-16T04:41:18.768002Z","shell.execute_reply.started":"2021-08-16T04:41:18.230088Z","shell.execute_reply":"2021-08-16T04:41:18.766993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindf = pd.read_csv('../input/siimcovidtfrecords/train.csv')\ntestdf = pd.read_csv('../input/siimcovidtfrecords/test.csv')\nnames = traindf.classLabel.to_list()\nlabels = traindf['class'].to_list()\nnames2labels = dict(set(zip(names,labels)))\nlabels2names = {v:k for k, v in names2labels.items()}","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:18.769959Z","iopub.execute_input":"2021-08-16T04:41:18.770367Z","iopub.status.idle":"2021-08-16T04:41:18.900600Z","shell.execute_reply.started":"2021-08-16T04:41:18.770324Z","shell.execute_reply":"2021-08-16T04:41:18.899855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS=5\nDEVICE=\"TPU\"\nIMG_SIZES = [[512,512]]*FOLDS\nBATCH_SIZES = [32]*FOLDS\nBATCH_SIZE = 32\nEPOCHS      = [12]*FOLDS\nPROBABILITY = 0.75\nCT          = 8\nSZ          = 0.08\nSEED=42","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:19.010899Z","iopub.execute_input":"2021-08-16T04:41:19.011251Z","iopub.status.idle":"2021-08-16T04:41:19.017905Z","shell.execute_reply.started":"2021-08-16T04:41:19.011218Z","shell.execute_reply":"2021-08-16T04:41:19.016849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dropout(image,DIM=IMG_SIZES[0], PROBABILITY = 0.6, CT = 5, SZ = 0.1):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*min(DIM),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3], dtype = image.dtype) \n        three = image[ya:yb,xb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0) \n\n    image = tf.reshape(image,[*DIM,3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:22.470609Z","iopub.execute_input":"2021-08-16T04:41:22.471047Z","iopub.status.idle":"2021-08-16T04:41:22.483870Z","shell.execute_reply.started":"2021-08-16T04:41:22.471009Z","shell.execute_reply":"2021-08-16T04:41:22.482876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://medium.com/mlait/affine-transformation-image-processing-in-tensorflow-part-1-df96256928a\n#ref:https://en.wikipedia.org/wiki/Rotation_matrix\n#ref:https://en.wikipedia.org/wiki/Shear_matrix\n#ref:https://en.wikipedia.org/wiki/Scaling_(geometry)\ndef getTransformMat(rotation, shear, yzoom, xzoom, yshift, xshift):\n        \n    rot = math.pi * rotation/180.\n    shear    = math.pi * shear/ 180.\n    \n    cos1   = tf.math.cos(rot)\n    sin1   = tf.math.sin(rot)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotMat = tf.reshape(tf.concat([[cos1,   sin1,   zero, -sin1,  cos1,   zero, zero, zero, one]],axis=0), [3,3])\n    cos2 = tf.math.cos(shear)\n    sin2 = tf.math.sin(shear)    \n    shearMat = tf.reshape(tf.concat([[one,  sin2,   zero, zero, cos2,   zero, zero, zero, one]],axis=0), [3,3])        \n    zoomMat = tf.reshape(tf.concat([[one/yzoom, zero, zero, zero, one/xzoom, zero,zero, zero, one]],axis=0), [3,3])    \n    shiftMat = tf.reshape(tf.concat([[one,  zero, yshift, zero, one,  xshift, zero, zero, one]],axis=0), [3,3])\n    \n    return K.dot(K.dot(rotMat, shearMat), K.dot(zoomMat,shiftMat))\n\ndef transform(image, DIM=IMG_SIZES[0]):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    \n    # fixed for non-square image thanks to Chris Deotte\n    \n    if DIM[0]!=DIM[1]:\n        pad = (DIM[0]-DIM[1])//2\n        image = tf.pad(image, [[0, 0], [pad, pad+1],[0, 0]])\n        \n    NEW_DIM = DIM[0]\n    \n    XDIM = NEW_DIM%2 #fix for size 331\n    \n    rot = 0.0 * tf.random.normal([1], dtype='float32')\n    shr = 2.0 * tf.random.normal([1], dtype='float32') \n    yzoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n    xzoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n    yshift = 8.0 * tf.random.normal([1], dtype='float32') \n    xshift = 8.0 * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = getTransformMat(rot,shr,yzoom,xzoom,yshift,xshift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(NEW_DIM//2, -NEW_DIM//2,-1), NEW_DIM)\n    y   = tf.tile(tf.range(-NEW_DIM//2, NEW_DIM//2), [NEW_DIM])\n    z   = tf.ones([NEW_DIM*NEW_DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -NEW_DIM//2+XDIM+1, NEW_DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([NEW_DIM//2-idx2[0,], NEW_DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n    \n    if DIM[0]!=DIM[1]:\n        image = tf.reshape(d,[NEW_DIM, NEW_DIM,3])\n        image = image[:, pad:DIM[1]+pad,:]\n    image = tf.reshape(image, [*DIM, 3])\n        \n    return image","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:23.584490Z","iopub.execute_input":"2021-08-16T04:41:23.584919Z","iopub.status.idle":"2021-08-16T04:41:23.607658Z","shell.execute_reply.started":"2021-08-16T04:41:23.584865Z","shell.execute_reply":"2021-08-16T04:41:23.606497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def readTrain(example):\n    tfrec_format = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_id': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], tf.one_hot(example['target'], 4)\n\n\ndef readTest(example, return_image_id):\n    tfrec_format = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_id': tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_id'] if return_image_id else 0","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:28.741397Z","iopub.execute_input":"2021-08-16T04:41:28.741757Z","iopub.status.idle":"2021-08-16T04:41:28.749214Z","shell.execute_reply.started":"2021-08-16T04:41:28.741725Z","shell.execute_reply":"2021-08-16T04:41:28.748212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepImage(img, augment=True, dim=IMG_SIZES[0]): \n    sat  = (0.7, 1.3)\n    cont = (0.8, 1.2)\n    bri  =  0.1\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32)\n    img = img/255.0\n    \n    if augment:\n        img = transform(img,DIM=dim)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, sat[0], sat[1])\n        img = tf.image.random_contrast(img, cont[0], cont[1])\n        img = tf.image.random_brightness(img, bri)     \n                      \n    img = tf.reshape(img, [*dim, 3])\n            \n    return img","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:31.331254Z","iopub.execute_input":"2021-08-16T04:41:31.331748Z","iopub.status.idle":"2021-08-16T04:41:31.340012Z","shell.execute_reply.started":"2021-08-16T04:41:31.331711Z","shell.execute_reply":"2021-08-16T04:41:31.339272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(fileids):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(fileid).group(1)) \n         for fileid in fileids]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:32.402924Z","iopub.execute_input":"2021-08-16T04:41:32.403477Z","iopub.status.idle":"2021-08-16T04:41:32.408806Z","shell.execute_reply.started":"2021-08-16T04:41:32.403428Z","shell.execute_reply":"2021-08-16T04:41:32.408079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_ids=True, batch_size=16, dim=IMG_SIZES[0]):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*2, seed=SEED)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(readTrain, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: readTest(example, return_image_ids), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgid_or_label: (prepImage(img, augment=augment, dim=dim), \n                                               imgid_or_label), \n                num_parallel_calls=AUTO)\n    if labeled and augment:\n        ds = ds.map(lambda img, label: (dropout(img, DIM=dim, PROBABILITY = PROBABILITY, CT = CT, SZ = SZ), label),\n                    num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:34.946225Z","iopub.execute_input":"2021-08-16T04:41:34.946809Z","iopub.status.idle":"2021-08-16T04:41:34.957698Z","shell.execute_reply.started":"2021-08-16T04:41:34.946744Z","shell.execute_reply":"2021-08-16T04:41:34.956978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    imgs, tars = batch\n    plt.figure(figsize=(size*5, 5))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        plt.title(f'class: {labels2names[tars[img_idx].numpy().argmax()]}', fontsize=15)\n        plt.imshow(imgs[img_idx,:, :, :])\n        plt.xticks([])\n        plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:36.600208Z","iopub.execute_input":"2021-08-16T04:41:36.600764Z","iopub.status.idle":"2021-08-16T04:41:36.607957Z","shell.execute_reply.started":"2021-08-16T04:41:36.600715Z","shell.execute_reply":"2021-08-16T04:41:36.607190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\nds = get_dataset(files_train, augment=True, shuffle=False, repeat=True,labeled=True,return_image_ids=True,\n                dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold])\nds = ds.unbatch().batch(20)\nbatch = next(iter(ds))\ndisplay_batch(batch, 5);","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:40.012548Z","iopub.execute_input":"2021-08-16T04:41:40.012949Z","iopub.status.idle":"2021-08-16T04:41:46.122087Z","shell.execute_reply.started":"2021-08-16T04:41:40.012915Z","shell.execute_reply":"2021-08-16T04:41:46.121335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef build_model(dim=IMG_SIZES[0], ef=0):\n    inp = tf.keras.layers.Input(shape=(*dim,3))\n    base = EFNS[ef](input_shape=(*dim,3),weights='imagenet',include_top=False)\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(64, activation = 'relu')(x)\n    x = tf.keras.layers.Dense(4,activation='softmax')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.01)\n    auc = tf.keras.metrics.AUC(curve='ROC',\n                               multi_label=True)\n    acc = tf.keras.metrics.CategoricalAccuracy()\n    f1  = tfa.metrics.F1Score(num_classes=4,average='macro',threshold=None)\n    model.compile(optimizer=opt,loss=loss,metrics=[auc, acc, f1])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:50.418633Z","iopub.execute_input":"2021-08-16T04:41:50.419031Z","iopub.status.idle":"2021-08-16T04:41:50.429678Z","shell.execute_reply.started":"2021-08-16T04:41:50.418997Z","shell.execute_reply":"2021-08-16T04:41:50.428314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_lr_callback(batch_size=8, plot=False):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n    if plot:\n        plt.figure(figsize=(10,5))\n        plt.plot(np.arange(EPOCHS[0]), [lrfn(epoch) for epoch in np.arange(EPOCHS[0])], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('learnig rate')\n        plt.title('Learning Rate Scheduler')\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\n_=get_lr_callback(BATCH_SIZES[0], plot=True )","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:51.970358Z","iopub.execute_input":"2021-08-16T04:41:51.970764Z","iopub.status.idle":"2021-08-16T04:41:52.132558Z","shell.execute_reply.started":"2021-08-16T04:41:51.970727Z","shell.execute_reply":"2021-08-16T04:41:52.131666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE      = 0\nDISPLAY_PLOT = True\n\nDEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\n# NUMBER OF FOLDS. USE 2, 5, 10\nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 512 \nIMG_SIZES = [[512, 512]]*FOLDS\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [16]*FOLDS\nEPOCHS      = [12]*FOLDS\n\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [7]*FOLDS\n\n# Augmentations\nAUGMENT   = True\nTRANSFORM = True\n\n# transformations\nROT_    = 0.0\nSHR_    = 2.0\nHZOOM_  = 8.0\nWZOOM_  = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\n# Dropout\nPROBABILITY = 0.75\nCT          = 8\nSZ          = 0.08\n\n#bri, contrast\nsat  = (0.7, 1.3)\ncont = (0.8, 1.2)\nbri  =  0.1\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:41:55.335595Z","iopub.execute_input":"2021-08-16T04:41:55.335985Z","iopub.status.idle":"2021-08-16T04:41:55.343326Z","shell.execute_reply.started":"2021-08-16T04:41:55.335940Z","shell.execute_reply":"2021-08-16T04:41:55.342579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFF_NETS = [7]*FOLDS\nTTA=1\n#DISPLAY_PLOT = True\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\noof_pred = []; oof_tar = []; oof_val = []; oof_f1 = []; oof_ids = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(num_train_files))):\n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in idxT])\n    np.random.shuffle(files_train);\n    files_valid = tf.io.gfile.glob([GCS_PATH + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))\n    print('#'*25); print('#### FOLD',fold+1)\n    print('#### Image Size: (%i, %i) | model: %s | batch_size %i'%\n          (IMG_SIZES[fold][0],IMG_SIZES[fold][1],EFNS[EFF_NETS[fold]].__name__,BATCH_SIZES[fold]*REPLICAS))\n    train_images = count_data_items(files_train)\n    val_images   = count_data_items(files_valid)\n    print('#### Training: %i | Validation: %i'%(train_images, val_images))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])\n    print('#'*25)   \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_auc', verbose=2, save_best_only=True,\n        save_weights_only=True, mode='max', save_freq='epoch')\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=True, shuffle=True, repeat=True,\n                dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n        epochs=EPOCHS[fold], \n        callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,repeat=False,dim=IMG_SIZES[fold]),\n        verbose = False,\n    )\n    \n    # Loading best model for inference\n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)  \n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF with TTA...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_ids=False,augment=True if TTA>1 else False,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*2)\n    ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/2/REPLICAS\n    pred = model.predict(ds_valid,steps=STEPS,verbose=2)[:TTA*ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,4,TTA),order='F'),axis=-1) )                 \n    \n    # GET OOF TARGETS AND idS\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n            labeled=True, return_image_ids=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n                labeled=False, return_image_ids=True)\n    oof_ids.append( np.array([img_id.numpy().decode(\"utf-8\") for img, img_id in iter(ds.unbatch())]))\n        \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1], average='macro')\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    fig, (ax1, ax2) = plt.subplots(2,1)\n    fig.set_figheight(15)\n    fig.set_figwidth(15)\n    #plt.figure(figsize=(15,5))\n    ax1.plot(np.arange(len(history.history['auc'])),history.history['auc'],'-o',label='Train auc',color='#ff7f0e')\n    ax1.plot(np.arange(len(history.history['auc'])),history.history['val_auc'],'-o',label='Val auc',color='#1f77b4')\n    x = np.argmax(history.history['val_auc'] ); y = np.max(history.history['val_auc'] )\n    xdist = ax1.get_xlim()[1] - ax1.get_xlim()[0]; ydist = ax1.get_ylim()[1] - ax1.get_ylim()[0]\n    ax1.scatter(x,y,s=200,color='#1f77b4'); ax1.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n    ax1.set_ylabel('auc',size=14); ax1.set_xlabel('Epoch',size=14)\n    ax1.set_title('Epoch vs Accuracy')\n    #ax1.legend(loc=2)\n    ax2.plot(np.arange(len(history.history['auc'])),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n    ax2.plot(np.arange(len(history.history['auc'])),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n    x = np.argmin(history.history['val_loss'] ); y = np.min(history.history['val_loss'] )\n    ydist = ax2.get_ylim()[1] - ax2.get_ylim()[0]\n    ax2.scatter(x,y,s=200,color='#d62728'); ax2.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n    ax2.set_ylabel('Loss',size=14)\n    ax2.set_xlabel('Epoch',size=14)\n    ax2.set_title('Epoch vs Loss')\n    ax2.legend(loc=3)\n    fig.savefig(f'fig{fold}.png')\n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:42:00.751243Z","iopub.execute_input":"2021-08-16T04:42:00.751760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nids = np.concatenate(oof_ids); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof, average='macro')\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ncolumns = ['image_id', 'fold']+[f'{idx}_true' for idx in range(4)] + [f'{idx}_pred' for idx in range(4)]\ndf_oof = pd.DataFrame(np.concatenate([ids[:,None], folds[:, 0:1], true, oof], axis=1), columns=columns)\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T01:56:36.266228Z","iopub.execute_input":"2021-08-16T01:56:36.266703Z","iopub.status.idle":"2021-08-16T01:56:36.434921Z","shell.execute_reply.started":"2021-08-16T01:56:36.266669Z","shell.execute_reply":"2021-08-16T01:56:36.434195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}