{"cells":[{"metadata":{},"cell_type":"markdown","source":"Based on: https://www.kaggle.com/agentauers/incredible-tpus-finetune-effnetb0-b6-at-once","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\"\n\nCFG = dict(\n    epochs = 50,\n    lr = 0.1,  # learning rate  \n    inp_size = 128, # input image size\n    net_count         =   7,\n    batch_size        =  16,\n    \n    read_size         = 128, \n    crop_size         = 128, \n    net_size          = 128,\n    rot               = 180.0,\n    shr               =   2.0,\n    hzoom             =   8.0,\n    wzoom             =   8.0,\n    hshift            =   8.0,\n    wshift            =   8.0,\n\n    optimizer         = 'adam',\n    label_smooth_fac  =   0.05,\n    \n    tta_steps         =  4  \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, random, re, math, time\nrandom.seed(a=42)\n\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n!pip install -q efficientnet\nimport efficientnet.tfkeras as efn\n\nimport PIL\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom tqdm import tqdm\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\nimport xgboost as xgb\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.utils import class_weight\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED=42\n\ndef seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)\n\nBASEPATH = \"../input/siim-isic-melanoma-classification\"\ndf_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))\ndf_test  = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))\ndf_sub   = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))\n\n\nGCS_PATH    = KaggleDatasets().get_gcs_path('melanoma-128x128')\nGCS_PATH1=KaggleDatasets().get_gcs_path('malignant-v2-128x128')\n#GCS_PATH_2019    = KaggleDatasets().get_gcs_path('isic2019-384x384-cc')\n\n#files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))\n#files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))\n\nfiles_train = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nmalignant= tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')\nfiles_train += malignant\n\n#files_valid= tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\n#files_valid+=files_valid[:3]\n\n#files_valid+=malignant\n\n#files_train+=GCS_PATH_2019 \n#files_train =files_train[3:]\n\n#VALIDATION_FILENAMES1=tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\n#VALIDATION_FILENAMES1=VALIDATION_FILENAMES1[:3]\n\n#VALIDATION_FILENAMES= tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')\n#VALIDATION_FILENAMES+=VALIDATION_FILENAMES1\n\n#files_valid=VALIDATION_FILENAMES\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### split to files_train ,files_valid (80% ,20%)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#files_train = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nfiles_train ,files_valid = train_test_split(files_train,test_size = 0.20,random_state = SEED)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train on images","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, cfg):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = cfg[\"read_size\"]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = cfg['rot'] * tf.random.normal([1], dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32') \n    w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        #'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return (example['image'], (example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return (example['image'], (example['sex'], example['age_approx'], example['anatom_site_general_challenge']), (example['image_name'] if return_image_name else 0))\n\ndef prepare_data(data, cfg=None, augment=True):\n    img = data[0]\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform(img, cfg)\n        img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n\n    else:\n        img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])\n                                   \n    img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])\n    img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])\n    \n    sex_oh = tf.one_hot(data[1][0], 2)\n    age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)\n    anatom_site_general_challenge = tf.one_hot(data[1][2], 7)\n    dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)\n    return (img, dense)\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\nNUM_TEST_IMAGES = count_data_items(files_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(NUM_TEST_IMAGES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n\n    ds = ds.map(lambda img, dense, imgname_or_label: (prepare_data((img, dense), augment=augment, cfg=cfg), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(cfg['batch_size'] * REPLICAS)\n    ds =ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## image from train data\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data[0][0], data[1]\n        #img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nds = get_dataset(files_train, CFG).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## images from the valid data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_valid, CFG).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train.diagnosis.unique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test of image augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"image before the augmentaion \")\nds = get_dataset(files_train, CFG).unbatch().take(1)   \nshow_dataset(200, 1, 1, ds)\nprint(\"image after the augmentaion \")\nds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)\nds = ds.take(1).cache().repeat()\nds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n# ds = ds.map(lambda img, target: (prepare_image(img, cfg=CFG, augment=True), target), \n#             num_parallel_calls=AUTO)\nds = ds.map(lambda img, dense, target: (prepare_data((img, dense), cfg=CFG, augment=True), target), \n            num_parallel_calls=AUTO)\nds = ds.take(12*5)\nds = ds.prefetch(AUTO)\n\nshow_dataset(64, 12, 5, ds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Images from the test data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, CFG, augment=True, repeat=True, \n                         labeled=False, return_image_names=False).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Scheduler\n\n*  source [get_cosine_schedule_with_warmup](https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ndef get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_train  = count_data_items(files_train) / (CFG['batch_size'] * REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_train =int(steps_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import kernel_tensorflow_utils as ktu\nlr_callback = ktu.LRSchedulers.FineTuningLR(\n    \n    lr_start=1e-5, lr_max=5e-5 * strategy.num_replicas_in_sync, lr_min=1e-5,\n    lr_rampup_epochs=5, lr_sustain_epochs=0, lr_exp_decay=0.8, verbose=1)\n\nplt.figure(figsize=(8, 5))\nlr_callback.visualize(steps_per_epoch=steps_train, epochs=40)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 5))\nlr_callback.visualize(steps_per_epoch=steps_train, epochs=40)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### [source lr_callback  ](https://www.kaggle.com/chankhavu/a-beginner-s-tpu-kernel-single-model-0-97)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nimport tensorflow as tf\n\ndef KerasFocalLoss(target, input):\n    \n    gamma = 2.\n    input = tf.cast(input, tf.float32)\n    \n    max_val = K.clip(-input, 0, 1)\n    loss = input - input * target + max_val + K.log(K.exp(-max_val) + K.exp(-input - max_val))\n    invprobs = tf.log_sigmoid(-input * (target * 2.0 - 1.0))\n    loss = K.exp(invprobs * gamma) * loss\n    \n    return K.mean(K.sum(loss, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nimport tensorflow as tf\n\n# Compatible with tensorflow backend\n\ndef focal_loss_f(gamma=2., alpha=.25):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_loss(alpha=0.25,gamma=2.0):\n    def focal_crossentropy(y_true, y_pred):\n        bce = K.binary_crossentropy(y_true, y_pred)\n        \n        y_pred = K.clip(y_pred, K.epsilon(), 1.- K.epsilon())\n        p_t = (y_true*y_pred) + ((1-y_true)*(1-y_pred))\n        \n        alpha_factor = 1\n        modulating_factor = 1\n\n        alpha_factor = y_true*alpha + ((1-alpha)*(1-y_true))\n        modulating_factor = K.pow((1-p_t), gamma)\n\n        # compute the final loss and return\n        return K.mean(alpha_factor*modulating_factor*bce, axis=-1)\n    return focal_crossentropy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## [source focal_loss](https://www.kaggle.com/c/santander-customer-transaction-prediction/discussion/83363)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate,multiply, LocallyConnected2D, Lambda)\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nimport keras\nfrom keras.models import Model\nfrom keras.activations import hard_sigmoid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32)\ny = tf.keras.backend.hard_sigmoid(x)\ny.numpy()\nfrom keras.activations import hard_sigmoid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def EFFB_7():\n    with strategy.scope():\n        \n        # meta data:\n        # https://www.kaggle.com/rajnishe/rc-fork-siim-isic-melanoma-384x384/notebook\n        \n        img_inp = tf.keras.layers.Input(shape = (CFG['inp_size'], CFG['inp_size'], 3), name = 'img_inp')\n        meta_inp = tf.keras.layers.Input(shape = (10), name = 'meta_inp')\n        eff ='6'\n        constructor = getattr(efn, f'EfficientNetB{eff}')\n        efnetb = constructor(weights = 'noisy-student', include_top = False)     \n        \n        # attention :\n        # https://www.kaggle.com/kmader/attention-on-pretrained-vgg16-for-bone-age/notebook\n        \n        pt_depth = efnetb.get_output_shape_at(0)[-1]\n        pt_features = efnetb(img_inp)\n        bn_features = tf.keras.layers.BatchNormalization()(pt_features)\n        attn_layer = tf.keras.layers.Conv2D(64, kernel_size = (1, 1), padding = 'same', activation = 'swish')(tf.keras.layers.Dropout(0.5)(bn_features))\n        attn_layer = tf.keras.layers.Conv2D(16, kernel_size = (1, 1), padding = 'same', activation = 'swish')(attn_layer)\n        attn_layer = tf.keras.layers.Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'swish')(attn_layer)\n        attn_layer = tf.keras.layers.Conv2D(1, kernel_size = (1, 1), padding = 'valid', activation = hard_sigmoid)(attn_layer)\n        up_c2_w = np.ones((1, 1, 1, pt_depth))\n        up_c2 = tf.keras.layers.Conv2D(pt_depth, kernel_size = (1, 1), padding = 'same',  activation = 'linear',  use_bias = False,    weights = [up_c2_w]  )\n        up_c2.trainable = False\n        attn_layer = up_c2(attn_layer)\n        mask_features = tf.keras.layers.multiply([attn_layer, bn_features])\n        gap_features = tf.keras.layers.GlobalAveragePooling2D()(mask_features)\n        gap_mask = tf.keras.layers.GlobalAveragePooling2D()(attn_layer)\n        \n        gap = tf.keras.layers.Lambda(lambda x: x[0] / x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n        gap_dr = tf.keras.layers.Dropout(0.5)(gap)\n        dr_steps = tf.keras.layers.Dropout(0.5)(tf.keras.layers.Dense(128, activation = 'swish')(gap_dr))\n        \n        meta_layer = tf.keras.layers.Dense(16)(meta_inp)\n        meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)\n        meta_layer = tf.keras.layers.Activation('swish')(meta_layer)\n        meta_layer = tf.keras.layers.Dropout(0.5)(meta_layer)\n        meta_layer = tf.keras.layers.Dense(8)(meta_inp)\n        meta_layer = tf.keras.layers.BatchNormalization()(meta_layer)\n        meta_layer = tf.keras.layers.Activation('swish')(meta_layer)\n        meta_layer = tf.keras.layers.Dropout(0.5)(meta_layer)\n        \n        concat = tf.keras.layers.concatenate([dr_steps, meta_layer])\n        concat = tf.keras.layers.BatchNormalization()(concat)\n        concat = tf.keras.layers.Dense(512, activation = 'swish')(concat)        \n        concat = tf.keras.layers.Dropout(0.5)(concat)\n        output = tf.keras.layers.Dense(2, activation ='softmax',dtype='float32')(concat)\n\n        model = tf.keras.models.Model(inputs = [img_inp, meta_inp], outputs = [output])\n\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_addons as tfa\nopt = tfa.optimizers.RectifiedAdam()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tfa.losses.WeightedKappa(num_classes=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INI_LR=1.e-4\nimport tensorflow.keras as K\n#opt = K.optimizers.Adam(lr=INI_LR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\nimport keras.backend as K\nimport numpy as np\nfrom prettytable import PrettyTable\nfrom prettytable import ALL\nfrom sklearn.metrics import f1_score\nfrom matplotlib import pyplot as plt\n\ndef f1(y_true, y_pred):\n    y_pred = K.round(y_pred)\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1 - K.mean(f1)\n\n\ndef fbeta_score_macro(y_true, y_pred, beta=1, threshold=0.1):\n\n    y_true = K.cast(y_true, 'float')\n    y_pred = K.cast(K.greater(K.cast(y_pred, 'float'), threshold), 'float')\n\n    tp = K.sum(y_true * y_pred, axis=0)\n    fp = K.sum((1 - y_true) * y_pred, axis=0)\n    fn = K.sum(y_true * (1 - y_pred), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = (1 + beta ** 2) * p * r / ((beta ** 2) * p + r + K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def f1_loss(predict, target):\n    loss = 0\n    lack_cls = target.sum(dim=0) == 0\n    if lack_cls.any():\n        loss += F.binary_cross_entropy_with_logits(\n            predict[:, lack_cls], target[:, lack_cls])\n    predict = torch.sigmoid(predict)\n    predict = torch.clamp(predict * (1-target), min=0.01) + predict * target\n    tp = predict * target\n    tp = tp.sum(dim=0)\n    precision = tp / (predict.sum(dim=0) + 1e-8)\n    recall = tp / (target.sum(dim=0) + 1e-8)\n    f1 = 2 * (precision * recall / (precision + recall + 1e-8))\n    return 1 - f1.mean() + loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = tfa.optimizers.RectifiedAdam()\nranger = tfa.optimizers.Lookahead(opt, \n                                  sync_period=6, \n                                  slow_step_size=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compile_new_model():    \n    with strategy.scope():\n        model=EFFB_7()\n        \n        # warm up model\n        #for layer in model.layers:\n           # layer.trainable = False\n\n        #for i in range(-3,0):\n            #model.layers[i].trainable = True\n            \n         # train all layers\n        #for layer in model.layers:\n            #layer.trainable = True \n        \n        #opt = tfa.optimizers.RectifiedAdam()\n        model.compile(\n            optimizer ='adam',\n            #loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO),\n            loss = 'sparse_categorical_crossentropy',\n            # metrics=['sparse_categorical_accuracy']\n            metrics = ['sparse_categorical_accuracy']\n        )\n\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### balnce the data (33125 of 1 & 33125 of 0 (upsampling))","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_category = pd.merge(train2019 , train , on=\"target\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntrain = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))\nprint('train: ', train.shape, '| unique ids:', sum(train['target'].value_counts()))\nX_train, X_val = train_test_split(train, test_size=.2, stratify=train['target'], random_state=SEED)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nlbl_value_counts = train['target'].value_counts()\nclass_weights = {i: max(lbl_value_counts) / v for i, v in lbl_value_counts.items()}\nprint('classes weigths:', class_weights)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(train.target),\n                                                 train.target)\nclass_weights = dict(enumerate(class_weights))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#class_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_path=\"/kaggle/working/effnet6_weights.best.hdf5\"\n\ncheckpoint = ModelCheckpoint(file_path, monitor='auc', verbose=1, save_best_only=True, mode='max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds_train     = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)\nsteps_train  = count_data_items(files_train) / (CFG['batch_size'] * REPLICAS)\n#ds_train     = ds_train.map(lambda img, label: (img, tuple([label] * CFG['net_count'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ds_train     = ds_train.map(lambda img, label: (img, tuple([label] * CFG['net_count'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import tensorflow as tf\n#train_set = tf.data.Dataset.from_tensor_slices(ds_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ds_t  = ds_train[ds_train['target'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tf.data.Dataset.from_tensor_slices(list(ds_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n# Class count\n\ncount_class_0, count_class_1 = df_train.target.value_counts()\n\n# Divide by class\ndf_class_0 = df_train[df_train['target'] == 0]\ndf_class_1 = df_train[df_train['target'] == 1]\ndf_class_1_over = df_class_1.sample(count_class_0, replace=True)\ndf_test_over = pd.concat([df_class_0, df_class_1_over], axis=0)\n\nprint('Random over-sampling:')\nprint(df_test_over.target.value_counts())\n\ndf_test_over.target.value_counts().plot(kind='bar', title='Count (target)');\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Buidling model...\")\nmodel = compile_new_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nfrom keras.utils.vis_utils import plot_model\nplot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install livelossplot\nfrom livelossplot import PlotLossesKeras\ncb=[PlotLossesKeras()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds_valid= get_dataset(files_valid , CFG, augment=True, shuffle=True, repeat=True)\nsteps_valid= count_data_items(files_valid) / (CFG['batch_size']*8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(steps_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WORKERS=2\n#class_weight=class_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    history      = model.fit(ds_train, \n                                 verbose          = 1,\n                                 steps_per_epoch  = steps_train, \n                                 epochs           = CFG['epochs'],\n                                 callbacks        = [lr_callback ,cb],\n                                 validation_data  = ds_valid,\n                                 workers=WORKERS, use_multiprocessing=True,\n                                 #class_weight=class_weights,\n                                 validation_steps  = steps_valid\n                                 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = compile_new_model()\n#model.load_weights(file_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_valid= count_data_items(files_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(steps_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG['batch_size'] = 256\n\ncnt_test   = count_data_items(files_valid)\nsteps      = cnt_test / (CFG['batch_size'] * REPLICAS) * CFG['tta_steps']\nds_testAug = get_dataset(files_valid, CFG, augment=True, repeat=True, \n                         labeled=False, return_image_names=False)\n\nprobs = model.predict(ds_testAug, verbose=1, steps=steps)\nprobs = np.stack(probs)\nprobs = probs2 = probs[:cnt_test * CFG['tta_steps'],0]\nprobs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=1)\nprobs = np.mean(probs, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cmdataset = get_dataset(files_valid, CFG, augment=True, repeat=True ,labeled=True, return_image_names=False) # since we are splitting the dataset and iterating separately on images and labels, order matters.\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_TEST_IMAGES))).numpy() # get everything as one batch\n#cm_probabilities = model.predict(images_ds)\n#cm_predictions = np.argmax(cm_probabilities, axis=-1)\nprint(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\nprint(\"Predicted labels: \", probs, probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(probs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport sklearn.metrics\ny_true =cm_correct_labels\ny_pred = probs\ntn, fp, fn, tp = confusion_matrix(y_true, y_pred.round()).ravel()\nspecificity = tn /(tn+fp)\nsensitivity=  tp/ (tp+fn)\nPrecision = tp/(tp+fp)\nRecall = tp/ (tp+fn)\nF1_Score = 2*(Recall * Precision) / (Recall + Precision)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Specificity : {:.3f}, Sensitivity: {:.3f}, F1_Score: {:.3f}'.format(specificity, sensitivity,F1_Score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.metrics\nprint(sklearn.metrics.classification_report(y_true, y_pred.round()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c_m=confusion_matrix(y_true, y_pred.round())\nprint(c_m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Specificity : {:.3f}, Sensitivity: {:.3f}, F1_Score: {:.3f}'.format(specificity, sensitivity,F1_Score))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}