{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# pepi\nimport re\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nimport math\nimport PIL\n\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, BatchNormalization, LeakyReLU, Dropout, Input\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import SGD, RMSprop, Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.data.experimental import AUTOTUNE\nfrom tensorflow.keras.metrics import AUC, BinaryAccuracy\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.keras.applications import DenseNet121, DenseNet201\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport pydicom \nfrom tqdm import tqdm\nimport missingno as msno \nfrom sklearn.metrics import roc_curve, roc_auc_score, auc\nimport shutil\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\n\nbase_dir = '/kaggle/input/siim-isic-melanoma-classification/'\nexternal_base_dir = '/kaggle/input/melanoma-external-malignant-256/'\nexternal_tfrec_base_dir = '../input/melanoma-256x256'\n\nsns.set(style=\"darkgrid\")\nrandom.seed(42)\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Initialize notebook parameters\n\nHere will will initializer the TPUs or GPUs depending on the setup.\n\nWe will also set up some constants, like image size.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'GPU'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_HEIGHT = 256\nIMG_WIDTH = 256\nN_CHANNELS = 3\nepochs = 20\nBATCH_SIZE = 8 * REPLICAS\nIMAGE_SIZE = [IMG_HEIGHT, IMG_WIDTH]\nIMAGE_RESIZE = [IMG_HEIGHT, IMG_WIDTH]\ninput_shape = (IMG_HEIGHT, IMG_WIDTH, N_CHANNELS)\nBALANCE_DATA = True\naug_data = True\nNETWORK_MODEL = 'DenseNet201'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Getting images ready for the model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Depending on the devices being used we need to change the data path.\n\nTPUs need to be fed from Google Cloud Storage","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == 'TPU':\n    files_dir = KaggleDatasets().get_gcs_path('melanoma-256x256')\nelse:\n    files_dir = external_tfrec_base_dir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(files_dir + '/train*.tfrec'),\n    test_size=0.1, random_state=42\n)\nTEST_FILENAMES = tf.io.gfile.glob(files_dir + '/test*.tfrec')\nprint('Train TFRecord Files:', len(TRAINING_FILENAMES))\nprint('Validation TFRecord Files:', len(VALID_FILENAMES))\nprint('Test TFRecord Files:', len(TEST_FILENAMES))\n\nsubmission_example = pd.read_csv(base_dir + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 180. * tf.random.normal([1],dtype='float32')\n    shr = 2. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    h_shift = 8. * tf.random.normal([1],dtype='float32') \n    w_shift = 8. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH])\n    image = tf.cast(image, tf.float32) / 255.0  \n    image = tf.reshape(image, [IMG_HEIGHT, IMG_WIDTH, 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef augmentation_pipeline(image, label):\n    \"\"\"\n    add augmentation functions here\n    \"\"\"\n    if aug_data:\n        image,_ = transform(image,label)\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_hue(image, 0.01)\n        image = tf.image.random_saturation(image, 0.7, 1.3)\n        image = tf.image.random_contrast(image, 0.8, 1.2)\n        image = tf.image.random_brightness(image, 0.1)\n        \n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE \nprint(\"Num of steps per epoch:\", STEPS_PER_EPOCH)\nprint(\"Num of steps per validation\", VALID_STEPS)\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nprint(\"Training data examples...\")\nds = train_dataset.unbatch().take(12*5)  \nshow_dataset(64, 12, 5, ds)\nprint(\"Validation data examples...\")\nds = valid_dataset.unbatch().take(12*5)  \nshow_dataset(64, 12, 5, ds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_roc(y_true, y_score):\n    \"\"\"\n    \"\"\"\n    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_true.ravel(), y_score.ravel())\n    roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n    \n    plt.figure()\n    lw = 2\n    plt.plot(fpr[2], tpr[2], color='darkorange',\n             lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \ndef plot_train_hist(hist):\n    # Plot training & validation accuracy values\n    plt.plot(hist.history['auc'])\n    plt.plot(hist.history['val_auc'])\n    plt.title('Model accuracy')\n    plt.ylabel('Auc')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()\n\n    # Plot training & validation loss values\n    plt.plot(hist.history['loss'])\n    plt.plot(hist.history['val_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.1 EfficientNet transfer learning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(base_dir + 'train.csv')\ny_train = train_df['target']\n\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 classes=np.unique(y_train),\n                                                 y=y_train)\n\n\n\nclass_weights = {0: class_weights[0],1: class_weights[1]}\nif not BALANCE_DATA:\n    class_weights = {0: 1,1: 2}\nprint(class_weights)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some callbacks for stopping the training process if validation loss stops decreasing. And a learning rate reduction during training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction =ReduceLROnPlateau( \n    monitor='loss',    # Quantity to be monitored.\n    factor=0.25,       # Factor by which the learning rate will be reduced. new_lr = lr * factor\n    patience=2,        # The number of epochs with no improvement after which learning rate will be reduced.\n    verbose=1,         # 0: quiet - 1: update messages.\n    mode=\"auto\",       # {auto, min, max}. In min mode, lr will be reduced when the quantity monitored has stopped decreasing; \n                       # in the max mode it will be reduced when the quantity monitored has stopped increasing; \n                       # in auto mode, the direction is automatically inferred from the name of the monitored quantity.\n    min_delta=0.0001,  # threshold for measuring the new optimum, to only focus on significant changes.\n    cooldown=0,        # number of epochs to wait before resuming normal operation after learning rate (lr) has been reduced.\n    min_lr=0.00001     # lower bound on the learning rate.\n    )\n\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 0.000003\n    lr_max     = 0.000020 * REPLICAS\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    \n    model_input = tf.keras.Input(shape=input_shape, name='input_layer')\n    dummy = tf.keras.layers.Lambda(lambda x:x)(model_input)    \n    outputs = []    \n\n    \n    x = DenseNet201(include_top=False, weights='imagenet', \n                    input_shape=input_shape, \n                    pooling='avg')(dummy)\n\n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    outputs.append(x)\n    \n    model = tf.keras.Model(model_input, outputs, name='transfNetwork')\n    model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the base pre-trained model\n\nmodel = get_model()\n\nmodel.compile(optimizer='adam', loss=BinaryCrossentropy(label_smoothing=0.05),\n              metrics=['AUC'])\n\n# train the model on the new data for a few epochs\nhist_train = model.fit(train_dataset,\n                 validation_data=valid_dataset,\n                 steps_per_epoch=STEPS_PER_EPOCH,\n                 validation_steps=VALID_STEPS,\n                 epochs=epochs,callbacks=[get_lr_callback()], class_weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_train_hist(hist_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Create predictions csv¶","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model.predict(test_images_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del submission_example['target']\nsubmission_example = submission_example.merge(pred_df, on='image_name')\n#sub.to_csv('submission_label_smoothing.csv', index=False)\nsubmission_example.to_csv('submission_densenet.csv', index=False)\nsubmission_example.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}