{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n# pepi\nimport re\nimport os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\nimport random\nimport math\nimport PIL\nimport tensorflow as tf, tensorflow.keras.backend as K\nimport tensorflow as tf\nfrom tensorflow.keras.applications.inception_v3 import InceptionV3\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D, BatchNormalization, LeakyReLU, Dropout, Input\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import SGD, RMSprop, Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.data.experimental import AUTOTUNE\nfrom tensorflow.keras.metrics import AUC, Accuracy\nfrom tensorflow.keras.losses import BinaryCrossentropy\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport pydicom \nfrom tqdm import tqdm\nimport missingno as msno \nfrom sklearn.metrics import roc_curve, roc_auc_score, auc\nimport shutil\nfrom sklearn.model_selection import train_test_split\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\n\nbase_dir = '/kaggle/input/siim-isic-melanoma-classification/'\nexternal_base_dir = '/kaggle/input/melanoma-external-malignant-256/'\nexternal_tfrec_base_dir = '../input/melanoma-256x256'\n\nsns.set(style=\"darkgrid\")\nrandom.seed(42)\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Initialize notebook parameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'TPU'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_HEIGHT = 256\nIMG_WIDTH = 256\nN_CHANNELS = 3\nepochs = 20\nBATCH_SIZE = 16 * REPLICAS\nIMAGE_SIZE = [IMG_HEIGHT, IMG_WIDTH]\nIMAGE_RESIZE = [IMG_HEIGHT, IMG_WIDTH]\ninput_shape = (IMG_HEIGHT, IMG_WIDTH, N_CHANNELS)\nBALANCE_DATA = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Getting images ready for the model\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == 'TPU':\n    files_dir = KaggleDatasets().get_gcs_path('melanoma-256x256')\nelse:\n    files_dir = external_tfrec_base_dir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(files_dir + '/train*.tfrec'),\n    test_size=0.1, random_state=42\n)\nTEST_FILENAMES = tf.io.gfile.glob(files_dir + '/test*.tfrec')\nprint('Train TFRecord Files:', len(TRAINING_FILENAMES))\nprint('Validation TFRecord Files:', len(VALID_FILENAMES))\nprint('Test TFRecord Files:', len(TEST_FILENAMES))\n\nsubmission_example = pd.read_csv(base_dir + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 90. * tf.random.normal([1],dtype='float32')\n    shr = 2. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    h_shift = 8. * tf.random.normal([1],dtype='float32') \n    w_shift = 8. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug_data = True\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0        \n    image = tf.reshape(image, [IMAGE_SIZE[0],IMAGE_SIZE[1], 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTO)\n    return dataset\n\ndef augmentation_pipeline(image, label):\n    \"\"\"\n    add augmentation functions here\n    \"\"\"\n    if aug_data:\n        image,_ = transform(image,label)\n        image = tf.image.random_flip_left_right(image)\n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE \nprint(\"Num of steps per epoch:\", STEPS_PER_EPOCH)\nprint(\"Num of steps per validation\", VALID_STEPS)\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nds = train_dataset.unbatch().take(12*5)  \nds_val = valid_dataset.unbatch().take(12*5)  \nshow_dataset(64, 12, 5, ds)\nshow_dataset(64, 12, 5, ds_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_roc(y_true, y_score):\n    \"\"\"\n    \"\"\"\n    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_true.ravel(), y_score.ravel())\n    roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n    \n    plt.figure()\n    lw = 2\n    plt.plot(fpr[2], tpr[2], color='darkorange',\n             lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n    \ndef plot_train_hist(hist):\n    # Plot training & validation accuracy values\n    plt.plot(hist.history['auc'])\n    plt.plot(hist.history['val_auc'])\n    plt.title('Model accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()\n\n    # Plot training & validation loss values\n    plt.plot(hist.history['loss'])\n    plt.plot(hist.history['val_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inception V3 transfer learning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(base_dir + 'train.csv')\ny_train = train_df['target']\n\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 classes=np.unique(y_train),\n                                                 y=y_train)\n\n\nclass_weights = {0: class_weights[0],1: class_weights[1]}\nif not BALANCE_DATA:\n    class_weights = {0: 1,1: 2}\nprint(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction =ReduceLROnPlateau( \n    monitor='loss',    # Quantity to be monitored.\n    factor=0.25,       # Factor by which the learning rate will be reduced. new_lr = lr * factor\n    patience=2,        # The number of epochs with no improvement after which learning rate will be reduced.\n    verbose=1,         # 0: quiet - 1: update messages.\n    mode=\"auto\",       # {auto, min, max}. In min mode, lr will be reduced when the quantity monitored has stopped decreasing; \n                       # in the max mode it will be reduced when the quantity monitored has stopped increasing; \n                       # in auto mode, the direction is automatically inferred from the name of the monitored quantity.\n    min_delta=0.0001,  # threshold for measuring the new optimum, to only focus on significant changes.\n    cooldown=0,        # number of epochs to wait before resuming normal operation after learning rate (lr) has been reduced.\n    min_lr=0.00001     # lower bound on the learning rate.\n    )\n\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)\n\ndef get_lr_callback():\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer_train = RMSprop(learning_rate=0.00025,\n    rho=0.9,\n    momentum=0.1,\n    epsilon=1e-07,\n    centered=True,\n    name='RMSprop')\n\noptimizer_fine = SGD(lr=0.000005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the base pre-trained model\nbase_model = InceptionV3(weights='imagenet', include_top=False,\n                         input_shape=(IMG_HEIGHT, IMG_WIDTH, N_CHANNELS))\n\n# add a global spatial average pooling layer\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\n\n# sigmoid for binary classification with one neuron\npredictions = Dense(1, activation='sigmoid')(x)\n\n# this is the model we will train\nmodel = Model(inputs=base_model.input, outputs=predictions)\n\n# first: train only the top layers (which were randomly initialized)\n# i.e. freeze all convolutional InceptionV3 layers\nfor layer in base_model.layers:\n    layer.trainable = False\n\n# compile the model\n# (should be done *after* setting layers to non-trainable)\nmodel.compile(optimizer=optimizer_train, loss=BinaryCrossentropy(label_smoothing=0.05),\n              metrics=['AUC'])\n\n# train the model on the new data for a few epochs\nhist_train = model.fit(train_dataset,\n                 validation_data=valid_dataset,\n                 steps_per_epoch=STEPS_PER_EPOCH,\n                 validation_steps=VALID_STEPS,\n                 epochs=epochs,callbacks=[learning_rate_reduction, es])\n\n#we chose to train the top 2 inception blocks, i.e. we will freeze\n# the first 249 layers and unfreeze the rest:\nfor layer in model.layers[:249]:\n    layer.trainable = False\nfor layer in model.layers[249:]:\n    layer.trainable = True\n\n# we need to recompile the model for these modifications to take effect\n# we use SGD with a low learning rate\n\nmodel.compile(optimizer=optimizer_fine,\n              loss=BinaryCrossentropy(label_smoothing=0.05),\n             metrics=['AUC'])\n\n# we train our model again (\n# this time fine-tuning the top 2 inception blocks\n# alongside the top Dense layers\nhist_fine = model.fit(train_dataset,\n                 validation_data=valid_dataset,\n                 steps_per_epoch=STEPS_PER_EPOCH,\n                 validation_steps=VALID_STEPS,\n                 epochs=epochs, callbacks=[get_lr_callback(), es])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_train_hist(hist_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_train_hist(hist_fine)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Create predictions csv¶","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probabilities = model.predict(test_images_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del submission_example['target']\nsubmission_example = submission_example.merge(pred_df, on='image_name')\n#sub.to_csv('submission_label_smoothing.csv', index=False)\nsubmission_example.to_csv('submission_b5.csv', index=False)\nsubmission_example.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}