{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport cv2\nimport re\nimport math\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Input, Dense, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, Callback\n\nimport tensorflow_addons as tfa\nfrom tensorflow_addons.optimizers import SWA, RectifiedAdam , Lookahead  \nfrom tensorflow_addons.losses import SigmoidFocalCrossEntropy\n\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import roc_curve, auc, roc_auc_score\nfrom sklearn.model_selection import train_test_split, KFold\n\n# Import EfficientNet\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.keras.backend.clear_session()\n    tf.tpu.experimental.shutdown_tpu_system()\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Determine Main Settings ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = efn.EfficientNetB3\nDIM = 256\nWHAT_PREDICT = 0\n\nUSE_ISIC2019 = True\nODD_OR_EVEN = 'odd'\n\nPL_TEST_ADD = False\nPL_TEST_ADD_MULTI = 2\n\nUPSAMPLE_FLG = False\nUPSAMPLING_MULTIPLAER = 2\n\nLOAD_WEIGHTS = False\n\nEPOCHS = 20\nN_FOLDS = 3\nWGTS = [1/N_FOLDS]*N_FOLDS\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nSEED_ALL = 298149\nSUBMISSION_NAME = 'submission.csv'\n\nLR_SCALE = 1\nTTA = 20\n\n# Coarse DropOut\nDROPRATE = 0\nDROPCT = 8\nDROPSIZE = 0.15\n\n# Chess Dropout\nCHESS_DROPOUT = 1\nCHESS_BLOCK_SHAPE = (8,15)\nCHESS_PROB_EVEN = 0.6\nCHESS_PROB_ODD = 0.6\nCHESS_PROB_ROTATE = 0.5\nCHESS_PROB_TO_CHANGE_PROBS = 0.5\n\n#Sprinkles\nSPRINKLES_CFG = {'num_holes': 180, 'side_length': 19,'sprinkles_mode':'normal','sprinkles_prob': 0}\n\n# Hair Augmentations\n# HAIR_NUM_MIN = 0\n# HAIR_NUM_MAX = 20\n# HAIR_AUG_PROB = 1.0\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download TFRecords Train/Isic2019/Test ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(f'cat-melanoma-{DIM}x{DIM}')\nGCS_DS_PATH_TEST = KaggleDatasets().get_gcs_path(f'melanoma-hair-{DIM}x{DIM}')\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/train_cat*.tfrec')))\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_TEST + '/test_pl_*.tfrec')))\n\nif PL_TEST_ADD:\n    GCS_DS_PATH_PL = KaggleDatasets().get_gcs_path(f'melanoma-hair-{DIM}x{DIM}')\n    files_train_pl = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_PL + '/test_pl_*.tfrec')))\n    \nif USE_ISIC2019:\n    files_train_isic_2019 = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/isic2019_cat*.tfrec')))\n    files_train_isic_2018 = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/isic2018_cat*.tfrec')))\n    if ODD_OR_EVEN == 'both':\n        files_train_isic_2019 = np.concatenate([files_train_isic_2019,files_train_isic_2018])\n#   ADD Only ISIC2019\n    elif ODD_OR_EVEN == 'odd':\n        files_train_isic_2019 = files_train_isic_2019\n#   ADD Only ISIC2018 and ISIC2017\n    else:\n        files_train_isic_2019 = files_train_isic_2018\n        \nif UPSAMPLE_FLG:\n    GCS_DS_PATH_UPSAMPLING = KaggleDatasets().get_gcs_path(f'malignant-v2-{DIM}x{DIM}')\n    files_upsampling_train = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_UPSAMPLING + '/train*.tfrec')))[:15]\n    files_upsampling_online = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_UPSAMPLING + '/train*.tfrec')))[15:30]\n    if USE_ISIC2019:\n        if ODD_OR_EVEN == 'both':\n            files_upsampling_isic2019 = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_UPSAMPLING + '/train*.tfrec')))[30:]\n        elif ODD_OR_EVEN == 'add':\n            files_upsampling_isic2019 = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_UPSAMPLING + '/train*.tfrec')))[30:][1::2]\n        else:\n            files_upsampling_isic2019 = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH_UPSAMPLING + '/train*.tfrec')))[30:][::2]\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_from_image = pd.read_csv(f'../input/melanoma-{DIM}x{DIM}/train.csv')\nif USE_ISIC2019:\n    train_df_isic_2019 = pd.read_csv(f'../input/isic2019-{DIM}x{DIM}/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Input Pipeline for Images ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\n@tf.function\ndef chess_dropout(img, shape_img, shape_block, \n                  prob_dropout ,prob_rotate=0.5, \n                  prob_even_block = 0.5, prob_odd_block = 0.5,\n                  prob_to_change_probs = 0.5\n                 ):\n    if tf.cast( tf.random.uniform([],0,1)>=prob_dropout, tf.int32)==1:\n        return img\n    \n    rotate_flag = tf.cast( tf.random.uniform([],0,1)<prob_rotate, tf.int32)\n    if rotate_flag==1:\n        zero_block = tf.zeros(shape=(shape_block[1],shape_block[0],3))\n    else:\n        zero_block = tf.zeros(shape=(shape_block[0],shape_block[1],3))\n    shape_zeros = zero_block.shape\n#     shape_img = img.shape\n    \n    P_CHANGE_FLG = tf.cast( tf.random.uniform([],0,1)<=prob_to_change_probs, tf.int32)\n    if P_CHANGE_FLG==1:\n        PROB_EVEN = prob_odd_block\n        PROB_ODD = prob_even_block\n    else:\n        PROB_EVEN = prob_even_block\n        PROB_ODD = prob_odd_block\n    \n    def random_block(i,j):\n        if tf.math.floormod((i+j),2) == 0:\n            P0  =  tf.math.less(tf.random.uniform([],0,1),PROB_EVEN)\n            return P0\n        else:\n            P1  =  tf.math.less(tf.random.uniform([],0,1),PROB_ODD)\n            return P1\n    \n#     Количество блоков с строке\n    num_in_row_block = tf.math.floordiv(shape_img[1], shape_zeros[1])\n    residual_row = tf.math.floormod(shape_img[1], shape_zeros[1])\n# Количество блоков в столбце\n    num_in_col_block = tf.math.floordiv(shape_img[0], shape_zeros[0])\n    residual_col = tf.math.floormod(shape_img[0], shape_zeros[0])\n\n    \n    img_row = tf.compat.v1.placeholder(tf.float32, name=  'img_row')\n    img_row = tf.cast(zero_block, tf.float32) \n    img_col = tf.compat.v1.placeholder(tf.float32, name = 'img_col')\n    img_col = tf.cast(zero_block, tf.float32) \n    img_rest_row = tf.compat.v1.placeholder(tf.float32, name = 'img_rest_row')\n    img_rest_row = tf.cast(zero_block, tf.float32) \n    \n    for j in range(num_in_col_block):\n        tf.autograph.experimental.set_loop_options(\n        shape_invariants=[(img_row, tf.TensorShape([None,None,3])),\n                         (img_col, tf.TensorShape([None,None,3])),\n                          (img_rest_row, tf.TensorShape([None,None,3])),\n                         ])\n        for i in range(num_in_row_block):\n            if i==0:\n                img_row = (tf.cast(zero_block, tf.float32) \n                                       if random_block(i,j) \n                                       else img[shape_zeros[0]*j:shape_zeros[0]*(j+1) ,\n                                                          shape_zeros[1]*i:shape_zeros[1]*(i+1) ,\n                                                          :] \n                          )\n            else:\n                img_row = tf.concat([img_row, (tf.cast(zero_block, tf.float32) \n                                               if random_block(i,j) \n                                                  else img[shape_zeros[0]*j:shape_zeros[0]*(j+1) ,\n                                                          shape_zeros[1]*i:shape_zeros[1]*(i+1) ,\n                                                          :] \n                                              )], axis=1)\n        \n        img_row = tf.concat( [img_row, (tf.cast(zero_block[:,:residual_row,:], tf.float32 )\n                                        if random_block(num_in_row_block,j) \n                                                 else img[shape_zeros[0]*j:shape_zeros[0]*(j+1),\n                                                          num_in_row_block*shape_zeros[1]:,:]  \n                                       )], \n                                                                 axis=1)\n        if j==0:\n            img_col = img_row\n        else:\n            img_col = tf.concat( [img_col, img_row], axis=0)\n            \n    for i in range(num_in_row_block):\n        tf.autograph.experimental.set_loop_options(\n        shape_invariants=[\n                          (img_rest_row, tf.TensorShape([None,None,3])),\n                         ])\n        if i==0:\n            img_rest_row = (tf.cast(zero_block[:residual_col,:,:] , tf.float32)\n                            if random_block(num_in_col_block,i) \n                                                 else img[shape_zeros[0]*num_in_col_block:,\n                                                          shape_zeros[1]*i:shape_zeros[1]*(i+1) ,\n                                                          :] \n                           )\n        else:\n            img_rest_row = tf.concat([img_rest_row, (tf.cast(zero_block[:residual_col,:,:], tf.float32 )\n                                                     if random_block(num_in_col_block,i) \n                                                 else img[shape_zeros[0]*num_in_col_block:,\n                                                          shape_zeros[1]*i:shape_zeros[1]*(i+1) ,\n                                                          :] \n                                                    )], axis=1)\n    img_rest_row = tf.concat([ img_rest_row, (tf.cast(zero_block[:residual_col,:residual_row,:], tf.float32)\n                                              if random_block(num_in_row_block,num_in_col_block) else\n                        img[shape_zeros[0]*num_in_col_block:,\n                            num_in_row_block*shape_zeros[1]:,:]\n                                             )], axis=1)\n    \n    img = tf.concat([img_col,img_rest_row], axis=0)\n    img = tf.reshape(img, [shape_img[0],shape_img[1],3])\n    return img\n    \n    \n    \ndef make_mask(num_holes,side_length,rows, cols, num_channels):\n        \"\"\"Builds the mask for all sprinkles.\"\"\"\n        row_range = tf.tile(tf.range(rows)[..., tf.newaxis], [1, num_holes])\n        col_range = tf.tile(tf.range(cols)[..., tf.newaxis], [1, num_holes])\n        r_idx = tf.random.uniform([num_holes], minval=0, maxval=rows-1,\n                                  dtype=tf.int32)\n        c_idx = tf.random.uniform([num_holes], minval=0, maxval=cols-1,\n                                  dtype=tf.int32)\n        r1 = tf.clip_by_value(r_idx - side_length // 2, 0, rows)\n        r2 = tf.clip_by_value(r_idx + side_length // 2, 0, rows)\n        c1 = tf.clip_by_value(c_idx - side_length // 2, 0, cols)\n        c2 = tf.clip_by_value(c_idx + side_length // 2, 0, cols)\n        row_mask = (row_range > r1) & (row_range < r2)\n        col_mask = (col_range > c1) & (col_range < c2)\n\n        # Combine masks into one layer and duplicate over channels.\n        mask = row_mask[:, tf.newaxis] & col_mask\n        mask = tf.reduce_any(mask, axis=-1)\n        mask = mask[..., tf.newaxis]\n        mask = tf.tile(mask, [1, 1, num_channels])\n        return mask\n    \ndef sprinkles(image, cfg = SPRINKLES_CFG): \n    num_holes = cfg['num_holes']\n    side_length = cfg['side_length']\n    mode = cfg['sprinkles_mode']\n    PROBABILITY = cfg['sprinkles_prob']\n    \n    RandProb = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n    if (RandProb == 0)|(num_holes == 0): \n        return image\n    \n    img_shape = tf.shape(image)\n    if mode is 'normal':\n        rejected = tf.zeros_like(image)\n    elif mode is 'salt_pepper':\n        num_holes = num_holes // 2\n        rejected_high = tf.ones_like(image)\n        rejected_low = tf.zeros_like(image)\n    elif mode is 'gaussian':\n        rejected = tf.random.normal(img_shape, dtype=tf.float32)\n    else:\n        raise ValueError(f'Unknown mode \"{mode}\" given.')\n        \n    rows = img_shape[0]\n    cols = img_shape[1]\n    num_channels = img_shape[-1]\n    if mode is 'salt_pepper':\n        mask1 = make_mask(num_holes,side_length,rows, cols, num_channels)\n        mask2 = make_mask(num_holes,side_length,rows, cols, num_channels)\n        filtered_image = tf.where(mask1, rejected_high, image)\n        filtered_image = tf.where(mask2, rejected_low, filtered_image)\n    else:\n        mask = make_mask(num_holes,side_length,rows, cols, num_channels)\n        filtered_image = tf.where(mask, rejected, image)\n    return filtered_image    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef dropout(image, DIM=256, PROBABILITY = 0.75, CT = 8, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=DIM):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n#         'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n#         'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n#         'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n#         'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n#         'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], tf.cast((example['target']==WHAT_PREDICT), tf.int64)\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=256, droprate=0, dropct=0, dropsize=0):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n#     img = tf.keras.applications.efficientnet.preprocess_input(img, data_format=None)\n    \n    if augment:\n        img = transform(img,DIM=dim)\n#         if (droprate!=0)&(dropct!=0)&(dropsize!=0): \n#             img = dropout(img, DIM=dim, PROBABILITY=droprate, CT=dropct, SZ=dropsize) \n#         if HAIR_AUG_PROB!=0:\n#             img = hair_aug_tf(img, PROBABILITY = HAIR_AUG_PROB)\n#         if CHESS_DROPOUT!=0:\n#             img = chess_dropout(img, shape_img = (DIM,DIM),shape_block = CHESS_BLOCK_SHAPE,\n#                                prob_dropout =CHESS_DROPOUT, prob_rotate = CHESS_PROB_ROTATE,\n#                                 prob_even_block = CHESS_PROB_EVEN, prob_odd_block = CHESS_PROB_ODD,\n#                                 prob_to_change_probs = CHESS_PROB_TO_CHANGE_PROBS\n#                                )    \n        if SPRINKLES_CFG['sprinkles_prob'] >0:\n            img = sprinkles(img, SPRINKLES_CFG)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n                      \n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=BATCH_SIZE, dim=256):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        if type(repeat)==int:\n            ds = ds.repeat(repeat)\n        else:\n            ds = ds.repeat()\n#         options = tf.data.Options()\n#         options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.FILE\n#         ds = ds.with_options(options)\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim, \n                                                droprate=DROPRATE, dropct=DROPCT, dropsize=DROPSIZE), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n#     ds = strategy.experimental_distribute_dataset(ds)\n\n    return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define Model and LR Schedule ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(dim=DIM):\n#     with strategy.scope():\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n#     loss = SigmoidFocalCrossEntropy(alpha=0.75, gamma=2.0)\n    \n    model = tf.keras.Sequential([\n            MODEL(\n                input_shape=(dim, dim, 3),\n#                 weights='imagenet',\n                drop_connect_rate=0.4,\n                weights='noisy-student',\n                include_top=False\n            ),\n            tf.keras.layers.GlobalAveragePooling2D(),\n    #         Dense(256, activation='relu'),\n            Dropout(0.2),\n            Dense(1, activation='sigmoid')\n    ])\n\n    model.compile(\n            optimizer='adam',\n    #         optimizer=optimizer,\n#             loss = 'binary_crossentropy',\n           loss = loss,\n            metrics=tf.keras.metrics.AUC(name='training_AUC')\n    )\n    return model\n#         model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=BATCH_SIZE):\n    lr_start   = 0.000005*LR_SCALE\n    lr_max     = 0.00000125 * batch_size*LR_SCALE\n    lr_min     = 0.0000005*LR_SCALE\n    lr_ramp_ep = 4\n    lr_sus_ep  = 2\n    lr_decay   = 0.7\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Stratified KFold ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = KFold(n_splits = N_FOLDS, random_state=SEED_ALL, shuffle=True)\nif USE_ISIC2019:\n    cv_isic_2019 = KFold(n_splits = N_FOLDS, random_state=SEED_ALL, shuffle=True)\n    \noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor num_fold,split in enumerate(cv.split(files_train)):\n    train_idx = split[0]\n    valid_idx = split[1]\n    files_temp_train = files_train[train_idx]\n    files_temp_valid = files_train[valid_idx]\n    \n#     Добавляем PL\n    if PL_TEST_ADD:\n        files_temp_train = np.concatenate([files_temp_train,\n                                          np.tile(files_train_pl[train_idx], PL_TEST_ADD_MULTI)\n                                          ])\n        files_temp_valid = np.concatenate([files_temp_valid,\n                                          np.tile(files_train_pl[valid_idx], PL_TEST_ADD_MULTI)\n                                          ])\n    \n    \n    #     Добавляем Upsample\n    if UPSAMPLE_FLG:\n        files_temp_train = np.concatenate([files_temp_train, \n                                           np.tile(files_upsampling_train[train_idx],UPSAMPLING_MULTIPLAER),\n                                           np.tile(files_upsampling_online[train_idx],UPSAMPLING_MULTIPLAER)\n                                          ])\n        files_temp_valid = np.concatenate([files_temp_valid, \n                                           np.tile(files_upsampling_train[valid_idx],UPSAMPLING_MULTIPLAER),\n                                           np.tile(files_upsampling_online[valid_idx],UPSAMPLING_MULTIPLAER)\n                                          ])\n    # Добавляем Внешние данные с Upsample или Нет\n    if USE_ISIC2019:\n        split_isic = next(cv_isic_2019.split(files_train_isic_2019))\n        files_isic_temp_train = files_train_isic_2019[split_isic[0]]\n        files_isic_temp_valid = files_train_isic_2019[split_isic[1]]\n        if UPSAMPLE_FLG:\n            files_isic_temp_train = np.concatenate([files_isic_temp_train, np.tile(files_upsampling_isic2019[split_isic[0]],UPSAMPLING_MULTIPLAER)])\n            files_isic_temp_valid = np.concatenate([files_isic_temp_valid, np.tile(files_upsampling_isic2019[split_isic[1]],UPSAMPLING_MULTIPLAER)])\n        files_temp_train = np.concatenate([files_temp_train,files_isic_temp_train])\n        files_temp_valid = np.concatenate([files_temp_valid,files_isic_temp_valid])\n    # Обучаем модели\n    \n#     Remain Only ISIC2019\n#     files_temp_train = np.array([i for i in files_temp_train if 'isic' in i])\n    \n    train_dataset = get_dataset(files_temp_train, repeat=True, shuffle=True, augment=True,\n                                dim=DIM, batch_size=BATCH_SIZE, labeled=True, return_image_names=False)\n    valid_dataset = get_dataset(files_temp_valid, repeat=2, shuffle=False, augment=True, \n                                dim=DIM, batch_size=BATCH_SIZE*4,labeled=True, return_image_names=False)\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        f'{DIM}_fold_{num_fold}.h5', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=DIM)\n        \n    if LOAD_WEIGHTS:\n        model.load_weights(f'../input/weights-for-tuning-melanoma/{DIM}_fold_{num_fold}.h5')\n        print(f'Load Weights For Fold {num_fold}')\n    else:\n        history = model.fit(\n                                train_dataset,\n                                epochs=EPOCHS,\n                                callbacks=[sv, get_lr_callback(BATCH_SIZE)],\n                                validation_data = valid_dataset,\n                                steps_per_epoch = count_data_items(files_temp_train)//BATCH_SIZE\n                               )\n\n        print('Loading best model...')\n        model.load_weights(f'{DIM}_fold_{num_fold}.h5')\n    \n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF with TTA...')\n    valid_dataset = get_dataset(files_temp_valid, repeat=True, shuffle=False, augment=True,\n                                dim=DIM, batch_size=BATCH_SIZE*4, labeled=False,return_image_names=False)\n    ct_valid = count_data_items(files_temp_valid); \n    STEPS = int(round(TTA * ct_valid/BATCH_SIZE/4)+1)\n    pred = model.predict(valid_dataset,steps=STEPS)[:TTA*ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n    \n    # GET OOF TARGETS AND NAMES\n    valid_dataset = get_dataset(files_temp_valid, repeat=False, shuffle=False, augment=False,\n                                dim=DIM, batch_size=BATCH_SIZE*4, labeled=True, return_image_names=True)\n    \n    oof_tar.append( np.array([target.numpy() for img, target in iter(valid_dataset.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*num_fold )\n    valid_dataset = get_dataset(files_temp_valid, repeat=False, shuffle=False, augment=False,\n                                dim=DIM, batch_size=BATCH_SIZE*4, labeled=False, return_image_names=True)\n#     ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n#                 labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(valid_dataset.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    test_dataset = get_dataset(files_test, repeat=True, shuffle=False, augment=True, \n                               dim=DIM, batch_size=BATCH_SIZE*4, labeled=False, return_image_names=False)\n    ct_test = count_data_items(files_test); \n    STEPS = int(round(TTA * ct_test/BATCH_SIZE/4)+1)\n    pred = model.predict(test_dataset,steps=STEPS)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[num_fold]\n    \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_training_AUC'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(num_fold,oof_val[-1],auc))\n#     print('#### FOLD %i OOF AUC with TTA = %.3f'%(num_fold,auc))\n    del history\n    \n    \n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate OOF CV AUC ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); \ntrue = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); \nfolds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(\n    image_name = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if USE_ISIC2019:\n    df_oof_no_isic2019 = df_oof.set_index('image_name').join(train_df_from_image.set_index('image_name')['patient_id'])\n    df_oof_no_isic2019 = df_oof_no_isic2019[df_oof_no_isic2019['patient_id'].notnull()].drop('patient_id', axis=1)\n    print('Overall OOF AUC with TTA on internal = %.3f' % roc_auc_score(df_oof_no_isic2019['target'],df_oof_no_isic2019['pred']))\n    df_oof_no_isic2019.to_csv('oof_no_isic.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit DataSet ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=DIM,\n                 labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}