{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -ltr ../input/siim6balancedfoldsimageset/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"fold_file_path = '../input/siim6balancedfoldsimageset/'\n\n# File name follow same pattern\nfold_name = 'siim-fold-' #1.csv\n\n# For which Fold dataset to run\n# total number of folds are 6\n# each file has ~ 5.2K images \n# so that images are balanced\nFOLD_NUMBER = 1\n\n# Read the file of specified fold to run\ntrain = pd.read_csv(fold_file_path + fold_name + str(FOLD_NUMBER) + '.csv')\n\n#SEED value\nSEED_VALUE = 2244","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(train.info())\n#print(train.head(553))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_train = train[train.tfrecord != -1 ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx_list = []\nfor img_name in train.image_name.values:\n    if img_name.endswith('downsampled'):\n        idx = filter_train.index[filter_train['image_name'] == img_name].to_list()\n        #print(str(idx) + str(len(idx)) + ':' +img_name )\n        if len(idx) == 1:\n            idx_list.append(idx[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_train = filter_train.drop(idx_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt = 0\nfor img_name in filter_train.image_name.values:\n#    if img_name == -1:\n#        cnt = cnt + 1\n#        print(train[])\n\n    if img_name.endswith('downsampled'):\n        cnt = cnt + 1\nprint(cnt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport math\n\nfrom matplotlib import pyplot as plt\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\n\nimport efficientnet.tfkeras as efn\n\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nGCS_PATH_19 = KaggleDatasets().get_gcs_path('jpeg-isic2019-384x384')\nGCS_PATH_20 = KaggleDatasets().get_gcs_path('jpeg-melanoma-384x384')\n\n# Configuration\nEPOCHS = 6\nBATCH_SIZE = 4 * strategy.num_replicas_in_sync\nimg_size = 384\nIMAGE_SIZE = [img_size,img_size]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (GCS_PATH_19)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(filter_train.loc[train.image_name == 'ISIC_0000013'].year.to_numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_gcs_path(image_id):\n    \n    year_nb = filter_train.loc[train.image_name == image_id].year.to_numpy()\n    #print(year_nb)\n    GCS_PATH = ''\n    \n    if year_nb == 2019:\n        GCS_PATH = GCS_PATH_19 + '/train/' + image_id + '.jpg'\n    else:\n        GCS_PATH = GCS_PATH_20 + '/train/' + image_id + '.jpg'\n    \n    return GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_train[\"image_path\"] = filter_train[\"image_name\"].apply(lambda x : add_gcs_path(x))\nfilter_train[\"image_jpg_id\"] = filter_train[\"image_name\"].apply(lambda x: x + '.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shuffle the rows\nfilter_train = filter_train.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_train.head(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtrain, xval, ytrain, yval = train_test_split(filter_train[\"image_path\"], filter_train[\"target\"], \n                                              test_size = 0.10, stratify = filter_train[\"target\"],\n                                              random_state=SEED_VALUE)\n\ndf_train = pd.DataFrame({\"image_path\":xtrain, \"target\":ytrain})\ndf_val = pd.DataFrame({\"image_path\":xval, \"target\":yval})\n\ndf_train[\"target\"] = df_train[\"target\"].astype('int')\ndf_val[\"target\"] = df_val[\"target\"].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(df_train.target.value_counts())\n#print(df_val.target.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = df_train.image_path.values\nval_paths   = df_val.image_path.values\n\ntrain_labels = df_train.target\nval_labels   = df_val.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    if 0.5 > tf.random.uniform([1], minval = 0, maxval = 1):\n        rot = 15. * tf.random.normal([1],dtype='float32')\n    else:\n        rot = 180. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return image , label\n    #return {'inp1': tf.reshape(d,[DIM,DIM,3]), 'inp2': image['inp2']}, label\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n\ndef decode_image(filename, label=None, image_size=(img_size, img_size)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, size = image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef int_div_round_up(a, b):\n    return (a + b - 1) // b","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.transpose(image)\n    image = tf.image.rot90(image)\n    image = tf.image.random_saturation(image, 0.7, 1.3)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    image = tf.image.random_brightness(image, 0.1)    \n    # used in Christ's notebook\n    #image = tf.image.random_saturation(image, 0, 2)\n    #imgage = tf.image.random_contrast(img, 0.8, 1.2)\n    #imgage = tf.image.random_brightness(img, 0.1)\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    #.map(transform, num_parallel_calls = AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((val_paths, val_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nNUM_TRAINING_IMAGES = df_train.shape[0]\nNUM_VALIDATION_IMAGES = df_val.shape[0]\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = .00003\n\ndef get_model():\n    with strategy.scope():\n        img_input = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n        \n        base = efn.EfficientNetB5(weights = 'imagenet', include_top = False)\n        #base = tf.keras.applications.ResNet152V2(weights = 'imagenet', include_top = False)\n        \n        x = base(img_input)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        #x = tf.keras.layers.Dropout(0.3)(x)\n       #x = tf.keras.layers.Dense(128, activation = 'relu')(x)\n    \n        output = tf.keras.layers.Dense(1, activation = 'sigmoid')(x)\n        \n        model = tf.keras.models.Model(inputs = img_input, outputs = output)\n        \n        #opt = tf.keras.optimizers.Adam(learning_rate = LR)\n        #tfa.losses.SigmoidFocalCrossEntropy(gamma = 2.0, alpha = 0.80)\n        opt = tfa.optimizers.RectifiedAdam(lr=LR)\n    \n        model.compile(optimizer = opt, loss = 'binary_crossentropy', metrics = [ tf.keras.metrics.AUC() ] )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fold_1 = get_model()\nprint(model_fold_1.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lr scheduler\ncb_lr_schedule = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', factor = 0.4, \n                                                      patience = 2, verbose = 1, min_delta = 0.0001, mode = 'max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = model_fold_1.fit(\n    train_dataset, \n    epochs = EPOCHS, \n    callbacks = [cb_lr_schedule],\n    steps_per_epoch = STEPS_PER_EPOCH,\n    validation_data = valid_dataset,\n    validation_steps = VALIDATION_STEPS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history1.history['auc']\nval_acc = history1.history['val_auc']\n\nloss = history1.history['loss']\nval_loss = history1.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fold_1.save('B5_Fold1_balance.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}