{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\n!pip install image-classifiers==1.0.0b1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os , math , re , random\n\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import classification_report\nfrom sklearn.utils import shuffle\n\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nfrom classification_models.tfkeras import Classifiers\nfrom tensorflow.keras.callbacks import EarlyStopping,LearningRateScheduler, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras.models import Model\n\n\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.applications import DenseNet121, DenseNet201\nfrom tensorflow.keras.applications import vgg16\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.applications.resnet50 import ResNet50\nfrom tensorflow.keras.applications import MobileNet , MobileNetV2\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras import optimizers\n\n!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa\n\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for reproducible results :\ndef seed_everything(seed=13):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    os.environ['TF_KERAS'] = '1'\n    random.seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try :\n    tpu=tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on :',tpu.master())\nexcept ValueError :\n    tpu = None\n\nif tpu :    \n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse :\n    strategy = tf.distribute.get_strategy()\n    \nprint('Replicas :',strategy.num_replicas_in_sync)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO  = tf.data.experimental.AUTOTUNE\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\nEPOCHS = 5\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync \nimg_size = 512\nSEED =  42\nnb_classes = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/alaska2-image-steganalysis/sample_submission.csv')\nfiles_name = np.array(os.listdir('../input/alaska2-image-steganalysis/Cover'))\n\npath = '../input/alaska2-image-steganalysis/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#function to be able to read images\ndef append_path(pre) :\n    return np.vectorize(lambda file : os.path.join(GCS_DS_PATH,pre,file))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def append_path2(pre) :\n    return np.vectorize(lambda file : os.path.join(path,pre,file))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading file names and shuffling them\npositives = files_name.copy()\nnegatives = files_name.copy()\n\nnp.random.shuffle(positives)\nnp.random.shuffle(negatives)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_paths = append_path('Test')(sub.Id.values)\ntest_paths2 = append_path2('Test')(sub.Id.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creating data so that i have 30k pos image (10k of each transformation) and 30k neg image\njmipod = append_path('JMiPOD')(positives[:74999])\n#jmipod2 = append_path2('JMiPOD')(positives[:2000])\njuniward = append_path('JUNIWARD')(positives[:74999])\n#juniward2 = append_path2('JUNIWARD')(positives[2000:4000])\nuerd = append_path('UERD')(positives[:74999])\n#uerd2 = append_path2('UERD')(positives[4000:6000])\n\npos_paths = np.concatenate([jmipod,juniward,uerd])\n#pos_paths2 = np.concatenate([jmipod2,juniward2,uerd2])\nneg_paths = append_path('Cover')(negatives[:74999])\n#neg_paths2 = append_path2('Cover')(negatives[:6000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = np.concatenate([pos_paths,neg_paths])\n#train_paths2 = np.concatenate([pos_paths2,neg_paths2])\n\ntrain_labels = np.array([1] * len(pos_paths) + [0] * len(neg_paths))\ntrain_paths , train_labels = shuffle(train_paths,train_labels)\n#train_paths2 , train_labels = shuffle(train_paths2,train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting data into train 85% / validation 15%\nX_train , X_val, y_train , y_val = train_test_split(train_paths, train_labels , random_state=SEED,test_size = 0.15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data augmentation :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"bool_random_brightness = False\nbool_random_contrast = False\nbool_random_hue = False\nbool_random_saturation = False\n\ncutmix_rate = 0\nmixup_rate = 0\ngridmask_rate = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Converting to Ycbcr :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"! git clone https://github.com/dwgoon/jpegio\n# Once downloaded install the package\n!pip install jpegio/.\nimport jpegio as jio","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport jpegio as jpio\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This code extract YCbCr channels from a jpeg object\ndef JPEGdecompressYCbCr(jpegStruct):\n    \n    nb_colors=len(jpegStruct.coef_arrays)\n        \n    [Col,Row] = np.meshgrid( range(8) , range(8) )\n    T = 0.5 * np.cos(np.pi * (2*Col + 1) * Row / (2 * 8))\n    T[0,:] = T[0,:] / np.sqrt(2)\n    \n    sz = np.array(jpegStruct.coef_arrays[0].shape)\n    \n    imDecompressYCbCr = np.zeros([sz[0], sz[1], nb_colors]);\n    szDct = (sz/8).astype('int')\n    \n    \n    \n    for ColorChannel in range(nb_colors):\n        tmpPixels = np.zeros(sz)\n    \n        DCTcoefs = jpegStruct.coef_arrays[ColorChannel];\n        if ColorChannel==0:\n            QM = jpegStruct.quant_tables[ColorChannel];\n        else:\n            QM = jpegStruct.quant_tables[1];\n        \n        for idxRow in range(szDct[0]):\n            for idxCol in range(szDct[1]):\n                D = DCTcoefs[idxRow*8:(idxRow+1)*8 , idxCol*8:(idxCol+1)*8]\n                tmpPixels[idxRow*8:(idxRow+1)*8 , idxCol*8:(idxCol+1)*8] = np.dot( np.transpose(T) , np.dot( QM * D , T ) )\n        imDecompressYCbCr[:,:,ColorChannel] = tmpPixels;\n    return imDecompressYCbCr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, img in enumerate(os.listdir('../input/alaska2-image-steganalysis/Cover')[:10]):\n    imgRGB = mpimg.imread('../input/alaska2-image-steganalysis/Cover/' + img)\n    jpegStruct = jpio.read('../input/alaska2-image-steganalysis/Cover/' + img)\n    print(type(jpegStruct))\n    \n    imDecompressYCbCr = JPEGdecompressYCbCr(jpegStruct)\n    \n    print(type(imDecompressYCbCr))\n    print(imDecompressYCbCr.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''def decode_image(file,label=None,img_size=(img_size,img_size)) :\n    \n    func = tf.py_function(JPEGdecompressYCbCr,[file],[tf.float16,tf.float16,3])\n    #bits = tf.io.read_file(file)\n    #image = tf.image.decode_jpeg(bits, channels = 3)\n    #image = tf.cast(image, tf.float16) # /255.0\n    #image = np.float32(image)\n    #image = image.eval(session=tf.compat.v1.Session())\n    #image= cv2.cvtColor(image.numpy(), cv2.COLOR_BGR2YCR_CB) \n    #image = tf.convert_to_tensor(image, dtype = tf.float16)\n    image = func(file)\n    print('done')\n    print(image.shape)\n    print(image)\n    image = tf.image.resize(image,img_size)\n    \n    if label is None :\n        return image\n    else :\n        return image,label'''\ndef decode_image(filename,label=None, image_size=(img_size,img_size)) :\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits,channels=3)\n    image = tf.cast(image, tf.float16) / 255.0\n    image = tf.image.resize(image,image_size)\n    if label == None :\n        return image\n    else :\n        return image, label\n    \ndef data_augment(image ,label = None,seed=2020) :\n    \n    image = tf.image.random_flip_left_right(image,seed=seed)\n    image = tf.image.random_flip_up_down(image,seed=seed)\n    if bool_random_brightness:\n        image = tf.image.random_brightness(image,0.2,seed=seed)\n    if bool_random_contrast:\n        image = tf.image.random_contrast(image,0.6,1.4, seed=seed)\n    if bool_random_hue:\n        image = tf.image.random_hue(image,0.07,seed=seed)\n    if bool_random_saturation:\n        image = tf.image.random_saturation(image,0.5,1.5,seed=seed)\n    \n    if label is None :\n        return image\n    else :\n        return image , label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CutMix :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch\ndef cutmix(image, label, PROBABILITY = cutmix_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZE):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MixUp :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixup(image, label, PROBABILITY = mixup_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = img_size\n    \n    imgs = []; labs = []\n    for j in range(BATCH_SIZE):\n        \n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,BATCH_SIZE),tf.int32)\n        a = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n\n        #mixup\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        if P==1:\n            a=0.\n        \n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        \n        # MAKE CUTMIX LABEL\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,nb_classes))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## GridMask :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n        # transform to radian\n        angle = math.pi * angle / 180\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform(image, rot_mat_inv, image_shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        return image*mask\n    else:\n        return image\n\ndef gridmask(img_batch, label_batch):\n    return apply_grid_mask(img_batch, (img_size,img_size, 3)), label_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Data object :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_train_data(train_paths,train_labels) :\n    train_data =  (\n    tf.data.Dataset.from_tensor_slices((train_paths,train_labels))\n    .map(decode_image, num_parallel_calls = AUTO)\n    .map(data_augment ,num_parallel_calls = AUTO)\n    .cache()\n    .repeat()\n    .shuffle(1024)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n    )    \n    \n    if cutmix_rate :\n        train_data = train_data.map(cutmix,num_parallel_calls = AUTO) \n    if mixup_rate : \n        train_data = train_data.map(mixup, num_parallel_calls = AUTO)\n    if gridmask_rate :\n        train_data = train_data.map(gridmask, num_parallel_calls = AUTO)\n    \n    return train_data\n\ndef create_validation_data(valid_paths,valid_labels):\n    valid_data = (\n        tf.data.Dataset.from_tensor_slices((valid_paths,valid_labels))\n        .map(decode_image , num_parallel_calls = AUTO)\n        .map(data_augment , num_parallel_calls = AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n        .cache()\n    )\n    return valid_data\n\ndef create_test_data(test_paths,aug=False):\n    test_data = (\n        tf.data.Dataset.from_tensor_slices(test_paths)\n        .map(decode_image, num_parallel_calls = AUTO)\n        .batch(BATCH_SIZE)\n        .prefetch(AUTO)\n    )\n    \n    if aug == True :\n        test_data=test_data.map(data_augment ,num_parallel_calls = AUTO)\n    return test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = tf.cast(train_labels,tf.float32)\ntrain_data = create_train_data(train_paths2,train_labels)\ntest_data = create_test_data(test_paths2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = tf.cast(train_labels,tf.float32)\ntrain_data = create_train_data(train_paths,train_labels)\ntest_data = create_test_data(test_paths)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Schedulers and Callbacks :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_start = 0.001\n\nlr_max = 0.001 * strategy.num_replicas_in_sync\nlr_min = 0.001 \nlr_rampup_epochs = 1\nlr_sustain_epochs = 2\nlr_exp_decay = .8\n\n\ndef lrfn(epoch) :\n    if epoch < lr_rampup_epochs :\n        lr = lr_start + (lr_max-lr_min) / lr_rampup_epochs * epoch\n    elif epoch < lr_rampup_epochs + lr_sustain_epochs :\n        lr = lr_max\n    else :\n        lr = lr_min + (lr_max - lr_min) * lr_exp_decay**(epoch - lr_sustain_epochs - lr_rampup_epochs)\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nfrom matplotlib import pyplot as plt\n\nplt.plot(rng,y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler = LearningRateScheduler(lrfn , verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mc = ModelCheckpoint('best_model.h5',monitor=tf.keras.metrics.AUC(),mode='max',save_best_only=True,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor=tf.keras.metrics.AUC(),mode='max',verbose=1,patience=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Weighted AUC metric for Alaska :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/anokas/weighted-auc-metric-updated\nfrom sklearn import metrics\nimport numpy as np\n\ndef alaska_weighted_auc(y_true, y_valid):\n    tpr_thresholds = [0.0, 0.4, 1.0]\n    weights =        [       2,   1]\n    \n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)\n    \n    # size of subsets\n    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\n    \n    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.\n    normalization = np.dot(areas, weights)\n    \n    competition_metric = 0\n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min # normalize such that curve starts at y=0\n        score = metrics.auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n        \n    return competition_metric / normalization","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"focal_loss = False\nlabel_smoothing = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_generalized(name,trainable_layers=20):\n    if name == 'EfficientNet' :\n        base_model = efn.EfficientNetB7(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )\n    if name == 'EfficientNet0' :\n        base_model = efn.EfficientNetB0(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                   )\n    if name == 'EfficientNet1' :\n        base_model = efn.EfficientNetB2(weights='imagenet',\n                                        include_top = False,\n                                        input_shape=(img_size,img_size,3)\n                                       )    \n    elif name == 'DenseNet' :\n        base_model = DenseNet201(weights='imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'MobileNet' :\n        base_model = MobileNet(weights = 'imagenet', include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'Inception' :\n        base_model = InceptionV3(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'ResNet' :\n        base_model = ResNet50(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3))\n    elif name == 'Incepresnet' :\n        base_model = InceptionResNetV2(weights = 'imagenet',include_top=False,input_shape=(img_size,img_size,3)) \n    \n    elif name == 'SEResNet50' :\n        seresnet50, _ = Classifiers.get('seresnet50')\n        base_model =  seresnet50(weights = 'imagenet', include_top = False, input_shape = (img_size,img_size,3))\n    elif name == 'SEResNext50' :\n        seresnext50 , _ = Classifiers.get('seresnext50')\n        base_model = seresnext50(weights = 'imagenet', include_top = False,input_shape = (img_size,img_size,3))\n    elif name == 'NasNetLarge' :\n        nasnet , _ = Classifiers.get('nansnetlarge')\n        base_model = nasnet(waights= 'imagenet', include_top = False , input_shape = (img_size,img_size,3))\n        \n    base_model.trainable = True\n    for layer in base_model.layers[:-trainable_layers] :\n        layer.trainable = True\n    layer = base_model.output\n    layer = L.GlobalAveragePooling2D()(layer)\n    layer = L.Dense(1024,activation='relu')(layer)\n    layer = L.Dropout(0.3)(layer,training=True)\n    predictions = L.Dense(nb_classes,activation='sigmoid')(layer)\n    predictions = tf.cast(predictions,tf.float32)\n    model = Model(inputs = base_model.input, outputs=predictions)\n    if focal_loss : \n        loss= tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    if label_smoothing :\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)\n    else :\n        loss = 'binary_crossentropy'\n    \n    #opt = optimizers.Adam(learning_rate = 0.001)\n    \n    model.compile(optimizer='adam',loss=loss,metrics=['accuracy',tf.keras.metrics.AUC()])  \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_effnet = get_model_generalized('EfficientNet')\nmodel_effnet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  '''model = tf.keras.Sequential([\n        base_model,\n        L.GlobalAveragePooling2D(),\n       # L.Dense(256,activation='relu'),\n       # L.Dropout(0.4),\n        L.Dense(1, activation='sigmoid')\n    ])'''\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = len(train_labels) // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model_effnet = get_model_generalized('EfficientNet1')\nhistory = model_effnet.fit(\n    train_data,\n    steps_per_epoch = steps_per_epoch,\n   # validation_data = valid_data,\n    epochs = EPOCHS,\n    #callbacks = [lr_scheduler]  # early stopping , Model checkpoint , schedulers\n    )    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model_effnet.predict(test_data , verbose=1)\nsub['Label'] = predictions\nsub.to_csv('e2_fulldata.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model_effnet \nimport gc \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope() :\n    model_incepresnet = get_model_generalized('DenseNet')\nhistory = model_incepresnet.fit(\n    train_data,\n    steps_per_epoch = steps_per_epoch,\n    epochs = EPOCHS,\n    #callbacks = []   early stopping , Model checkpoint , schedulers\n    ) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model_incepresnet.predict(test_data , verbose=1)\nsub['Label'] = predictions\nsub.to_csv('DenseNetBaseline.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling With Cross Validation : ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"kfolds = 5\nprobs = []\nhistories = [] \n\nfolds = KFold (kfolds , shuffle=True , random_state=SEED)\n\nfor i,(train_indices,valid_indices) in enumerate(folds.split(train_paths,train_labels)) :\n    print('#'*20)\n    print('Fold :',i+1)\n    print('#'*20)\n    \n    trn = train_paths[train_indices]\n    valid = train_paths[valid_indices]\n    \n    trn_labels = train_labels[train_indices]\n    valid_labels = train_labels[valid_indices]\n    \n    with strategy.scope() :\n        model_crossval = get_model_generalized('EfficientNet')\n    history = model_crossval.fit(create_train_data(trn,trn_labels),\n                                 steps_per_epoch = trn.shape[0] // BATCH_SIZE,\n                                 validation_data = create_validation_data(valid,valid_labels),\n                                 epochs = EPOCHS,\n                                 verbose = 1,\n                                 #callbacks = [lr_scheduler,mc]\n                                )\n    prob = model_crossval.predict(test_data)\n    \n    histories.append(history)\n    probs.append(prob)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prob_sum = 0\nfor prob in probs :\n    prob_sum = prob_sum + prob\nprob_avg = prob_sum / kfolds\n\nsub['Label'] = prob_avg\nsub.to_csv('effnetCV', index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# With Test Time augmentation :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#model_effnet.load('best_model.h5')\n\ntta_num = 5\nprobabilities = []\nfor i in range(tta_num) :\n    print('TTA number :',i+1)\n    test_tta = create_test_data(test_paths)\n    prob = model_effnet.predict(test_tta)\n    probabilities.append(prob)\n    \n    \ntab = np.zeros((len(probabilities[1]),1))\nfor i in range(len(probabilities[1])) :\n    for j in range(tta_num) :\n        tab[i] += probabilities[j][i] \ntab = tab / tta_num\nsub['Label'] = tab\nsub.to_csv('model_name+TTA.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# One vs all approach :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = create_test_data(test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_model(steg):\n    pos_path_jmipod = append_path(steg)(positives[:60000])\n    neg_path_jmipod = append_path('Cover')(negatives[:60000])\n\n    train_paths = np.concatenate([pos_paths,neg_paths])\n    train_labels = np.array([1] * len(pos_paths) + [0] * len(neg_paths))\n\n    train_paths , train_labels = shuffle(train_paths,train_labels)\n    X_train , X_val, y_train , y_val = train_test_split(train_paths, train_labels , random_state=SEED,test_size = 0.15)\n\n    train_data_jmipod = create_train_data(X_train,y_train)\n    valid_data_jmipod = create_valid_data(X_val,y_val)\n\n    steps_per_epoch = X_train.shape[0] // BATCH_SIZE\n    with strategy.scope() :\n        model_effnet = get_model_generalized('EfficientNet')\n    history = model_effnet.fit(\n        train_data,\n        steps_per_epoch = steps_per_epoch,\n        validation_data = valid_data,\n        epochs = EPOCHS,\n        #callbacks = []   early stopping , Model checkpoint , schedulers\n        )\n    predictions = model_effnet.predict(test_data)\n    sub['Label'] = predictions\n    sub.to_csv('effnet'+steg+'.csv',index=False)\n\n    return sub\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_jmipod = binary_model('JMiPOD')\nsub_juniward = binary_model('JUNIWARD')\nsub_uerd = binary_model('UERD')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}