{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is my work throughout this competition you'll find evrything you need in here, it's based on chris's notebook so thanks to him. At the end you'll find how to create pseudo labels and tf records , load them and retrain for better score !","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2, pandas as pd, matplotlib.pyplot as plt\ntrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\nprint('Examples WITH Melanoma')\nimgs = train.loc[train.target==1].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    plt.subplot(2,5,i+1); plt.axis('off')\n    plt.imshow(img)\nplt.show()\nprint('Examples WITHOUT Melanoma')\nimgs = train.loc[train.target==0].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    img = cv2.imread('../input/jpeg-melanoma-128x128/train/%s.jpg'%k)\n    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n    plt.subplot(2,5,i+1); plt.axis('off')\n    plt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\n!pip install tensorflow-addons=='0.9.1'\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 47\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768 \nIMG_SIZES = [512,512,512,512,512]\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [0,0,0,0,0]\nINC2018 = [1,1,1,1,1]\nPSEUDO = [1,1,1,1,1]\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [32]*FOLDS\nEPOCHS = [17]*FOLDS\n\n\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [4,4,4,4,4]\nfocal_loss = 0\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# SET MIXED PRECISION\nMIXED_PRECISION = True\nXLA_ACCELERATE = True\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if DEVICE == 'TPU': policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS\nGCS_PATH3 = [None]  * FOLDS\n\nfor i in range(len(GCS_PATH3)) : \n    GCS_PATH3[i] = KaggleDatasets().get_gcs_path('melanoma-pseudo-labelling-tf-record-512x512') #pseudodata ../input/data-pseudo-tf-records-09576-09526\n\nfor i,k in enumerate(IMG_SIZES):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n    GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))\n\nfiles_pseulabeling = np.sort(np.array(tf.io.gfile.glob(GCS_PATH3[0] + '/train_pseudo_*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(files_pseulabeling)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cutmix_rate = 0.3\ngridmask_rate = 0.3\nimg_size = 512\n#BATCH_SIZE = 32\nnb_classes = 1\n\n\nDROP_FREQ = [0,0.75,0.75] # between 0 and 1\nDROP_CT = [0,8,8] # may slow training if CT>16\nDROP_SIZE = [0,0.2,0.2] # between 0 and 1\n\n\nRATE = 0; CT = 0; SIZE = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch\ndef cutmix_v2(data, label, PROBABILITY = cutmix_rate):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    \n    DIM = img_size    \n    imgs = []; labs = []\n    \n    for j in range(BATCH_SIZES[fold] * REPLICAS):\n        \n        #random_uniform( shape, minval=0, maxval=None)        \n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n        \n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast(tf.random.uniform([], 0, BATCH_SIZES[fold] * REPLICAS), tf.int32)\n        \n        # CHOOSE RANDOM LOCATION\n        x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)\n        \n        # Beta(1, 1)\n        b = tf.random.uniform([], 0, 1) # this is beta dist with alpha=1.0\n        \n\n        WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        \n        # MAKE CUTMIX IMAGE\n        one = data['image_data'][j,ya:yb,0:xa,:]\n        two = data['image_data'][k,ya:yb,xa:xb,:]\n        three = data['image_data'][j,ya:yb,xb:DIM,:]        \n        #ya:yb\n        middle = tf.concat([one,two,three],axis=1)\n\n        img = tf.concat([data['image_data'][j,0:ya,:,:],middle,data['image_data'][j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        \n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        lab1 = label[j,]\n        lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n\n    image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZES[fold] * REPLICAS ,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(BATCH_SIZES[fold] * REPLICAS, nb_classes))\n    \n    data['image_data'] = image2\n    \n    return data,label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ncfg = dict(\ntransform_prob    =  1.0,\nrot               = 180.0,\nshr               =   2.0,\nhzoom             =   8.0,\nwzoom             =   8.0,\nhshift            =   8.0,\nwshift            =   8.0,\n)\n\ndef transform_shear_rot(image,cfg):\n    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = img_size\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = cfg['rot'] * tf.random.normal([1],dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1],dtype='float32') \n    w_shift = cfg['wshift'] * tf.random.normal([1],dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_grid(image, inv_mat, image_shape):\n    h, w, c = image_shape\n    cx, cy = w//2, h//2\n    new_xs = tf.repeat( tf.range(-cx, cx, 1), h)\n    new_ys = tf.tile( tf.range(-cy, cy, 1), [w])\n    new_zs = tf.ones([h*w], dtype=tf.int32)\n    old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))\n    old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)\n    clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)\n    clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)\n    clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)\n    old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))\n    old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))\n    new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))\n    new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))\n    old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)\n    new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)\n    rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))\n    rotated_image_channel = list()\n    for i in range(c):\n        vals = rotated_image_values[:,i]\n        sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])\n        rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))\n    return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])\n\ndef random_rotate(image, angle, image_shape):\n    def get_rotation_mat_inv(angle):\n        # transform to radian\n        angle = math.pi * angle / 180\n        cos_val = tf.math.cos(angle)\n        sin_val = tf.math.sin(angle)\n        one = tf.constant([1], tf.float32)\n        zero = tf.constant([0], tf.float32)\n        rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)\n        rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])\n        return rot_mat_inv\n    angle = float(angle) * tf.random.normal([1],dtype='float32')\n    rot_mat_inv = get_rotation_mat_inv(angle)\n    return transform_grid(image, rot_mat_inv, image_shape)\n\ndef GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):\n    h, w = image_height, image_width\n    hh = int(np.ceil(np.sqrt(h*h+w*w)))\n    hh = hh+1 if hh%2==1 else hh\n    d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)\n    l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)\n\n    st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n    st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)\n\n    y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)\n    x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)\n\n    for i in range(0, hh//d+1):\n        s1 = i * d + st_h\n        s2 = i * d + st_w\n        y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)\n        x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)\n\n    x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)\n    y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)\n    clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)\n\n    x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))\n    y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))\n\n    hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)), tf.int32)])\n    x_ranges = tf.repeat(x_ranges, hh)\n    y_ranges = tf.repeat(y_ranges, hh)\n\n    y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))\n    x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))\n\n    y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64),  tf.zeros_like(y_ranges), [hh, hh])\n    y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)\n\n    x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])\n    x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)\n\n    mask = tf.expand_dims( tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)\n\n    mask = random_rotate(mask, rotate_angle, [hh, hh, 1])\n    mask = tf.image.crop_to_bounding_box(mask, (hh-h)//2, (hh-w)//2, image_height, image_width)\n\n    return mask\n\n\n\ndef gridmask_v2(data, label_batch):\n    return apply_grid_mask_v2(data, (img_size,img_size, 3)), label_batch\n\ndef apply_grid_mask_v2(data, image_shape, PROBABILITY = gridmask_rate):\n    AugParams = {\n        'd1' : 100,\n        'd2': 160,\n        'rotate' : 45,\n        'ratio' : 0.3\n    }\n    \n        \n    mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])\n    if image_shape[-1] == 3:\n        mask = tf.concat([mask, mask, mask], axis=-1)\n        mask = tf.cast(mask,tf.float32)\n        P = tf.cast(tf.random.uniform([], 0, 1) <= PROBABILITY, tf.int32)\n    if P==1:\n        data['image_data'] = data['image_data']  * mask\n        return data\n    else:\n        return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dropout(image, DIM=img_size, PROBABILITY = 0.75, CT = 8, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=img_size):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'patient_id'                   : tf.io.FixedLenFeature([], tf.int64),\n        'sex'                          : tf.io.FixedLenFeature([], tf.int64),\n        'age_approx'                   : tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n      #  'diagnosis'                    : tf.io.FixedLenFeature([], tf.int64),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    \n    data = {}\n    \n    image = prepare_image(example['image'])\n    data['patient_id'] = tf.cast(example['patient_id'],tf.float32)\n    data['sex'] = tf.cast(example['patient_id'],tf.float32)\n    data['age_approx'] = tf.cast(example['age_approx'],tf.float32)\n    #data['anatom_site_general_challenge'] = tf.cast(tf.one_hot(example['anatom_site_general_challenge'], 7), tf.int32)\n    \n    data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)\n    #data['source'] = tf.cast(example['source'],tf.float32)\n    \n    label = tf.cast(example['target'], tf.float32)   \n    \n    return image, label , data\n\n\ndef read_unlabeled_tfrecord(example,augment ,return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        \"patient_id\" : tf.io.FixedLenFeature([],tf.int64) ,\n      \"sex\": tf.io.FixedLenFeature([],tf.int64),\n      \"age_approx\": tf.io.FixedLenFeature([],tf.int64),\n      \"anatom_site_general_challenge\": tf.io.FixedLenFeature([],tf.int64),\n    }\n    \n\n    example = tf.io.parse_single_example(example, tfrec_format)\n    data = {}\n    image = prepare_image(example['image'],augment = augment)\n    img_name = example['image_name'] if return_image_name else 0\n    \n    data['patient_id'] = tf.cast(example['patient_id'],tf.float32)\n    data['sex'] = tf.cast(example['patient_id'],tf.float32)\n    data['age_approx'] = tf.cast(example['age_approx'],tf.float32)\n    \n    data['anatom_site_general_challenge'] = tf.cast(example['anatom_site_general_challenge'],tf.float32)\n    \n    return image, img_name , data\n    \n \ndef prepare_image(img, augment=True, dim=img_size):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        img = transform(img,DIM=dim)\n        if (RATE!=0)&(CT!=0)&(SIZE!=0): \n            img = dropout(img, DIM=dim, PROBABILITY=RATE, CT=CT, SZ=SIZE)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n      \n                      \n    img = tf.reshape(img, [dim,dim, 3])\n    img = tf.image.resize(img, [img_size,img_size])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_setup(image,label,data) :\n    #anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32) for i in range(7)]\n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']]#,'source','anatom_site_general_challenge',\n    tabular=tf.stack(tab_data) #+anatom\n    \n    return {'image_data' : image , 'meta_data' : tabular} , label\n\ndef test_setup(image,image_name,data) :\n    #anatom = [tf.cast(data['anatom_site_general_challenge'][i], dtype = tf.float32) for i in range(7)]\n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in ['sex','age_approx','patient_id','anatom_site_general_challenge']] #,'source','anatom_site_general_challenge',\n    tabular=tf.stack(tab_data) #+anatom\n    return {'image_data' : image , 'meta_data' : tabular } , image_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True,val = False, batch_size=32, dim=img_size):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n        ds = ds.map(train_setup , num_parallel_calls = AUTO)\n        ds = ds.batch(batch_size * REPLICAS)\n        ds = ds.prefetch(AUTO) \n        \n        if val == False :           \n            if cutmix_rate :\n                ds = ds.map(cutmix_v2,num_parallel_calls = AUTO) \n            if gridmask_rate:\n                ds = ds.map(gridmask_v2, num_parallel_calls=AUTO)\n      \n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example,augment, return_image_names), \n                    num_parallel_calls=AUTO)\n        ds = ds.map(test_setup , num_parallel_calls = AUTO)\n        ds = ds.batch(batch_size * REPLICAS)\n        ds = ds.prefetch(AUTO)\n        \n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" '''ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras.applications.resnet50 import ResNet50\n\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.applications import DenseNet121, DenseNet201\n\nEFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef build_model(dim=img_size, ef=0):\n    inp = tf.keras.layers.Input(shape=(dim,dim,3),name = 'image_data')\n    inp2 = tf.keras.layers.Input(shape=(4),name = 'meta_data')\n    base = EFNS[ef](input_shape=(dim,dim,3),weights='noisy-student',include_top=False , drop_connect_rate = 0.4) # try noisy student weights\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    #x = tf.keras.layers.Dense(1024, activation = 'relu')(x) #added\n    #x = tf.keras.layers.Dropout(0.4)(x) #added\n    #x = tf.keras.layers.Dense(512, activation = 'relu')(x) #added\n    #x = tf.keras.layers.Dropout(0.4)(x) #added\n    \n    x2 = tf.keras.layers.Dense(128, activation='relu')(inp2) #added\n    #x2 = tf.keras.layers.Dropout(0.4)(x2) #added\n    #x2 = tf.keras.layers.Dense(128, activation='relu')(x2) #added\n    #x2 = tf.keras.layers.Dropout(0.3)(x2) #added\n    #x2 = tf.keras.layers.Dense(32, activation='relu')(x2) #added\n    #x2 = tf.keras.layers.Dropout(0.3)(x2) #added\n    \n    \n    \n    \n    concat = tf.keras.layers.concatenate([x,x2]) #added\n    \n    concat = tf.keras.layers.Dense(512,activation = 'relu')(concat) #added\n    concat = tf.keras.layers.Dropout(0.4)(concat) #added\n    #concat = tf.keras.layers.Dense(256,activation = 'relu')(concat) #added#\n    #concat = tf.keras.layers.Dropout(0.3)(concat) #added\n    \n    \n    concat = tf.keras.layers.Dense(1,activation='sigmoid',dtype = 'float32')(concat)\n    \n    model = tf.keras.Model(inputs=[inp,inp2],outputs=concat)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    #opt = tf.train.experimental.enable_mixed_precision_graph_rewrite(opt)\n    if focal_loss : \n        loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)\n    else :    \n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 0\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n          (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n    if INC2019[fold]:\n        files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n        print('#### Using 2019 external data')\n    if INC2018[fold]:\n        files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])\n        print('#### Using 2018+2017 external data')\n    if PSEUDO[fold] : \n        files_train += tf.io.gfile.glob(GCS_PATH3[fold]+'/train_pseudo01-323.tfrec')\n        files_train += tf.io.gfile.glob(GCS_PATH3[fold]+'/train_pseudo00-1000.tfrec')\n        print('#### Using Pseudo Labelling')\n    np.random.shuffle(files_train); print('#'*25)\n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=img_size,ef=EFF_NETS[fold]) #IMG_SIZES[fold]\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=True, shuffle=True, repeat=True,\n                dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n        epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False,dim=IMG_SIZES[fold],val=True), #class_weight = {0:1,1:2},\n        verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF with TTA...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n    ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n    #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n            labeled=True, return_image_names=True,val=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n                labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n    \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'%\n                (fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)\n        plt.legend(loc=3)\n        plt.show()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(\n    image_name = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],\n                 labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('stratify_e4_128meta_512.4concat_18epochs_512img_with_pseudo_labeling_noisy_0.3gridmask_cutmix0.4drop_connect.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pseudo Labeling :","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntop = -100 #### Number of exapmles labeled as 1 with confidence\nbottom = 2000 ####Number of exapmles labeled as 0 with confidence\n\n\nmodel_1 = pd.read_csv('../input/data-for-pseudolabelling-09576-09526/averaging_bestsub_withmeta_data_0.75_0.25giba.csv')\nmodel_2 = pd.read_csv('../input/data-for-pseudolabelling-09576-09526/averaging_bestsub_withmeta_data_0.7_0.3giba.csv')\n#model_3 = pd.read_csv('../input/data-for-pseudolabelling-09576-09526/esemble_with_giba.csv')\n#model_4 = pd.read_csv('../input/data-for-pseudolabelling-09576-09526/esemblev2_with_giba.csv')\n#model_5 = pd.read_csv('../input/data-for-pseudolabelling-09576-09526/lghteams_models.csv')\n\ntest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n\nmodel_1.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel1 = model_1.sort_values(by = ['target'])[top:].image_name\nmodel1 = model1.values.tolist()\n\nmodel2 = model_2.sort_values(by = ['target'])[top:].image_name\nmodel2 = model2.values.tolist()\n\nmodel3 = model_3.sort_values(by = ['target'])[top:].image_name\nmodel3 = model3.values.tolist()\n\nmodel4 = model_4.sort_values(by = ['target'])[top:].image_name\nmodel4 = model4.values.tolist()\n\nmodel5 = model_5.sort_values(by = ['target'])[top:].image_name\nmodel5 = model5.values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_1_imagename = set(model1).intersection(model2) #.intersection(model3).intersection(model4).intersection(model5)\nprint('Number of new positive examples: %i' %len(label_1_imagename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_1_data = test.loc[test['image_name'].isin(label_1_imagename)].reset_index(drop = True)\nlabel_1_data['target'] = 1\nlabel_1_data.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = model_1.sort_values(by = ['target'])[:bottom].image_name\nmodel1 = model1.values.tolist()\n\nmodel2 = model_2.sort_values(by = ['target'])[:bottom].image_name\nmodel2 = model2.values.tolist()\n\nmodel3 = model_3.sort_values(by = ['target'])[:bottom].image_name\nmodel3 = model3.values.tolist()\n\nmodel4 = model_4.sort_values(by = ['target'])[:bottom].image_name\nmodel4 = model4.values.tolist()\n\nmodel5 = model_5.sort_values(by = ['target'])[:bottom].image_name\nmodel5 = model5.values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_0_imagename = set(model1).intersection(model2) #.intersection(model3).intersection(model4).intersection(model5)\nprint('Number of new negative examples: %i' %len(label_0_imagename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_0_data = test.loc[test['image_name'].isin(label_0_imagename)].reset_index(drop = True)\nlabel_0_data['target'] = 0\nlabel_0_data.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_1_data.dropna(inplace = True)\nlabel_0_data.dropna(inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat([label_0_data, label_1_data])\ndata.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/jpeg-melanoma-512x512/test/'\n\nIMGS = data['image_name'].values\nIMGS = IMGS + '.jpg'\n\nprint('Number of images: %i' %len(IMGS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"str_col = ['patient_id','sex','anatom_site_general_challenge'] \nfor col in str_col:\n    data[col], mp = data[col].factorize()\n    print(mp)\n\ndata['age_approx'] = data['age_approx'].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2, feature3, feature4, feature5, feature6):\n    \n    feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'patient_id': _int64_feature(feature2),\n      'sex': _int64_feature(feature3),\n      'age_approx': _int64_feature(feature4),\n      'anatom_site_general_challenge': _int64_feature(feature5),\n      'target': _int64_feature(feature6)\n  }\n    example_proto = tf.train.Example(features = tf.train.Features(feature = feature))\n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 1000\n\nCT = len(IMGS)//SIZE + int(len(IMGS)%SIZE != 0)\n\nfor j in range(CT):\n    print(); print('Writing TFRecord %i of %i...' %(j, CT))\n    CT2 = min(SIZE, len(IMGS)- j*SIZE)\n    \n    with tf.io.TFRecordWriter('train_pseudo_512%.2i-%i.tfrec' %(j, CT2)) as writer:\n        for k in range(CT2):\n            img = cv2.imread(PATH + IMGS[SIZE*j + k])\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 95))[1].tostring()\n            \n            name = IMGS[SIZE*j + k].split('.')[0]\n            row = data.loc[data.image_name == name]\n            \n            example = serialize_example(\n                img,\n                str.encode(name),\n                row.patient_id.values[0],\n                row.sex.values[0],\n                row.age_approx.values[0],                        \n                row.anatom_site_general_challenge.values[0],\n                row.target.values[0])\n            \n            writer.write(example)\n            \n            if k%100 == 0: print(k, ', ', end = '')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}