{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:11:54.710196Z","iopub.execute_input":"2021-06-14T07:11:54.710589Z","iopub.status.idle":"2021-06-14T07:12:01.932364Z","shell.execute_reply.started":"2021-06-14T07:11:54.710503Z","shell.execute_reply":"2021-06-14T07:12:01.931263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q chart_studio","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:01.933717Z","iopub.execute_input":"2021-06-14T07:12:01.933963Z","iopub.status.idle":"2021-06-14T07:12:07.574036Z","shell.execute_reply.started":"2021-06-14T07:12:01.933926Z","shell.execute_reply":"2021-06-14T07:12:07.572901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import modules\n\nimport numpy as np \nimport pandas as pd \nimport math\nimport os\nimport glob\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport PIL\n\n# plotly\nimport plotly.express as px\nimport chart_studio.plotly as py\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\nfrom matplotlib.image import imread\nimport cv2\n\nfrom sklearn.model_selection import ( KFold, train_test_split, \n                                    cross_validate, cross_val_score, GridSearchCV )\nfrom sklearn.metrics import ( roc_curve, auc, precision_recall_curve, \n                             average_precision_score, ConfusionMatrixDisplay )\nfrom mlxtend.plotting import plot_confusion_matrix\n\nfrom random import choices\nfrom functools import partial\nimport re\nfrom kaggle_datasets import KaggleDatasets\n\n# tensorflow\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import (Dense, Flatten, Dropout, LSTM, \n                                     Bidirectional, Lambda, Reshape,\n                                    GlobalAveragePooling2D) \nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nfrom keras.utils.vis_utils import plot_model\n\nimport efficientnet.tfkeras as efn\n\n# Suppress warnings \nimport warnings\nwarnings.simplefilter(action='ignore')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T10:57:53.578161Z","iopub.execute_input":"2021-06-14T10:57:53.578623Z","iopub.status.idle":"2021-06-14T10:57:53.605529Z","shell.execute_reply.started":"2021-06-14T10:57:53.578584Z","shell.execute_reply":"2021-06-14T10:57:53.604468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nsns.set(context=\"paper\", font=\"monospace\")\nsns.set(style=\"whitegrid\")\nplt.style.use('fivethirtyeight')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:10:25.527617Z","iopub.execute_input":"2021-06-14T11:10:25.528283Z","iopub.status.idle":"2021-06-14T11:10:25.537511Z","shell.execute_reply.started":"2021-06-14T11:10:25.528232Z","shell.execute_reply":"2021-06-14T11:10:25.536806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Data Loading and Exploration","metadata":{}},{"cell_type":"code","source":"# available files and folders \nprint(os.listdir(\"../input/siim-isic-melanoma-classification\"))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:16.453114Z","iopub.execute_input":"2021-06-14T07:12:16.453448Z","iopub.status.idle":"2021-06-14T07:12:16.467068Z","shell.execute_reply.started":"2021-06-14T07:12:16.453422Z","shell.execute_reply":"2021-06-14T07:12:16.466464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\n\ntest = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\n\nprint('Training data shape: ', train.shape)\ntrain.head() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:16.467895Z","iopub.execute_input":"2021-06-14T07:12:16.468125Z","iopub.status.idle":"2021-06-14T07:12:16.611560Z","shell.execute_reply.started":"2021-06-14T07:12:16.468101Z","shell.execute_reply":"2021-06-14T07:12:16.610651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train and test data info\nprint('Train Set')\nprint(train.info())\nprint('-------------')\nprint('Test Set')\nprint(test.info())","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:16.612847Z","iopub.execute_input":"2021-06-14T07:12:16.613105Z","iopub.status.idle":"2021-06-14T07:12:16.653591Z","shell.execute_reply.started":"2021-06-14T07:12:16.613079Z","shell.execute_reply":"2021-06-14T07:12:16.652976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Explore melanoma target","metadata":{}},{"cell_type":"code","source":"train['benign_malignant'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:16.655169Z","iopub.execute_input":"2021-06-14T07:12:16.655523Z","iopub.status.idle":"2021-06-14T07:12:16.664016Z","shell.execute_reply.started":"2021-06-14T07:12:16.655496Z","shell.execute_reply":"2021-06-14T07:12:16.663260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['benign_malignant'].value_counts(normalize=True).iplot(kind='bar',\n                                                      yTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='red',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Melanoma Target Distribution')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:10:34.358825Z","iopub.execute_input":"2021-06-14T11:10:34.359374Z","iopub.status.idle":"2021-06-14T11:10:34.426395Z","shell.execute_reply.started":"2021-06-14T11:10:34.359324Z","shell.execute_reply":"2021-06-14T11:10:34.425344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Highly imbalanced target. There are significantly more benign images compared to malignant images in the train dataset.","metadata":{}},{"cell_type":"markdown","source":"#### Sex distribution","metadata":{}},{"cell_type":"code","source":"train['sex'].value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:17.483040Z","iopub.execute_input":"2021-06-14T07:12:17.483362Z","iopub.status.idle":"2021-06-14T07:12:17.494051Z","shell.execute_reply.started":"2021-06-14T07:12:17.483336Z","shell.execute_reply":"2021-06-14T07:12:17.493211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['sex'].value_counts(normalize=True).iplot(kind='bar',\n                                                yTitle='Percentage', \n                                                linecolor='black', \n                                                opacity=0.7,\n                                                color='green',\n                                                theme='pearl',\n                                                bargap=0.3,\n                                                gridcolor='white',\n                                                title='Sex Distribution')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:17.495076Z","iopub.execute_input":"2021-06-14T07:12:17.495518Z","iopub.status.idle":"2021-06-14T07:12:17.623787Z","shell.execute_reply.started":"2021-06-14T07:12:17.495486Z","shell.execute_reply":"2021-06-14T07:12:17.623029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gender vs Target","metadata":{}},{"cell_type":"code","source":"z = train.groupby(['target','sex'])['benign_malignant'].count().to_frame().reset_index()\nz.style.background_gradient(cmap='Reds')  ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:17.624808Z","iopub.execute_input":"2021-06-14T07:12:17.625047Z","iopub.status.idle":"2021-06-14T07:12:17.674311Z","shell.execute_reply.started":"2021-06-14T07:12:17.625021Z","shell.execute_reply":"2021-06-14T07:12:17.673434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.catplot(x='target',y='benign_malignant', hue='sex',data=z,kind='bar')\nplt.ylabel('Count')\nplt.xlabel('benign:0 vs malignant:1');","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:17.675351Z","iopub.execute_input":"2021-06-14T07:12:17.675609Z","iopub.status.idle":"2021-06-14T07:12:18.038627Z","shell.execute_reply.started":"2021-06-14T07:12:17.675582Z","shell.execute_reply":"2021-06-14T07:12:18.037781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Location of imaged site","metadata":{}},{"cell_type":"code","source":"train['anatom_site_general_challenge'].value_counts(normalize=True).sort_values()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.039872Z","iopub.execute_input":"2021-06-14T07:12:18.040141Z","iopub.status.idle":"2021-06-14T07:12:18.050341Z","shell.execute_reply.started":"2021-06-14T07:12:18.040111Z","shell.execute_reply":"2021-06-14T07:12:18.049525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['anatom_site_general_challenge'].value_counts(normalize=True).sort_values().iplot(kind='barh',\n                                                      xTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='#FB8072',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Anatomical Site Distribution')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.051439Z","iopub.execute_input":"2021-06-14T07:12:18.051686Z","iopub.status.idle":"2021-06-14T07:12:18.110893Z","shell.execute_reply.started":"2021-06-14T07:12:18.051660Z","shell.execute_reply":"2021-06-14T07:12:18.110270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Age distribution of patients","metadata":{}},{"cell_type":"code","source":"train['age_approx'].iplot(kind='hist',bins=30,color='orange',xTitle='Age distribution',yTitle='Count')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.111862Z","iopub.execute_input":"2021-06-14T07:12:18.112228Z","iopub.status.idle":"2021-06-14T07:12:18.557942Z","shell.execute_reply.started":"2021-06-14T07:12:18.112181Z","shell.execute_reply":"2021-06-14T07:12:18.557242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Diagnosis Distribution","metadata":{}},{"cell_type":"code","source":"train['diagnosis'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.558868Z","iopub.execute_input":"2021-06-14T07:12:18.559200Z","iopub.status.idle":"2021-06-14T07:12:18.568262Z","shell.execute_reply.started":"2021-06-14T07:12:18.559173Z","shell.execute_reply":"2021-06-14T07:12:18.567482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['diagnosis'].value_counts(normalize=True).sort_values().iplot(kind='barh',\n                                                      xTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='blue',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Diagnosis Distribution')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.569197Z","iopub.execute_input":"2021-06-14T07:12:18.569638Z","iopub.status.idle":"2021-06-14T07:12:18.623930Z","shell.execute_reply.started":"2021-06-14T07:12:18.569611Z","shell.execute_reply":"2021-06-14T07:12:18.623036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualizing images","metadata":{}},{"cell_type":"code","source":"# Function for plotting samples\ndef plot_samples(samples):  \n    fig, axes = plt.subplots(nrows=4, ncols=5, figsize=(32,16))\n    for i in range(len(samples)):\n        image = imread(samples[i])\n        ax = axes[i//5][i%5]\n        ax.imshow(image)\n        if i<10: # first 10 files\n            ax.set_title(\"Benign\", fontsize=20)\n        else:\n            ax.set_title(\"Malignant\", fontsize=20)\n        ax.axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.625297Z","iopub.execute_input":"2021-06-14T07:12:18.625645Z","iopub.status.idle":"2021-06-14T07:12:18.631191Z","shell.execute_reply.started":"2021-06-14T07:12:18.625614Z","shell.execute_reply":"2021-06-14T07:12:18.630138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample images\n\ndirname = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\nsample_img = []\n\n# get 10 benign image files \nbenign_ = train[train['benign_malignant'] == 'benign']['image_name'][:10]\n\n# get 10 malignant image files \nmalignant_ = train[train['benign_malignant'] == 'malignant']['image_name'][:10]\n\n# get benign filepaths\nfor i in benign_:\n    sample_img.append(dirname + i + '.jpg')\n\n# get malignant filepaths \nfor j in malignant_:\n    sample_img.append(dirname + j + '.jpg')\n\nplot_samples(sample_img)\nplt.suptitle('Melanoma Samples', fontsize=30)\n# plt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:12:18.632455Z","iopub.execute_input":"2021-06-14T07:12:18.632730Z","iopub.status.idle":"2021-06-14T07:12:50.462659Z","shell.execute_reply.started":"2021-06-14T07:12:18.632696Z","shell.execute_reply":"2021-06-14T07:12:50.461562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Data Preparation and Augmentation","metadata":{}},{"cell_type":"markdown","source":"#### Connect to TPU","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is set. \n    # On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy() \n\nREPLICAS = strategy.num_replicas_in_sync\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T07:13:14.318018Z","iopub.execute_input":"2021-06-14T07:13:14.318327Z","iopub.status.idle":"2021-06-14T07:13:19.771493Z","shell.execute_reply.started":"2021-06-14T07:13:14.318301Z","shell.execute_reply":"2021-06-14T07:13:19.770520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Set parameters","metadata":{}},{"cell_type":"code","source":"SEED = 42\nFOLDS = 3\nEFF_NETS = 6 \nBATCH_SIZES = [bs * strategy.num_replicas_in_sync for bs in [32]*FOLDS] \nIMG_SIZES = [384]*FOLDS\nEPOCHS = 12 \nLR = 0.00004\nLABEL_SMOOTHING = 0.05\nTTA = 15 # test time augmentation","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:45:18.782855Z","iopub.execute_input":"2021-06-14T11:45:18.783202Z","iopub.status.idle":"2021-06-14T11:45:18.788453Z","shell.execute_reply.started":"2021-06-14T11:45:18.783171Z","shell.execute_reply":"2021-06-14T11:45:18.787218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seed to make sure the same random numbers are generated on multiple executions\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:45:28.493467Z","iopub.execute_input":"2021-06-14T11:45:28.493851Z","iopub.status.idle":"2021-06-14T11:45:28.499372Z","shell.execute_reply.started":"2021-06-14T11:45:28.493819Z","shell.execute_reply":"2021-06-14T11:45:28.498224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get Dataset","metadata":{}},{"cell_type":"code","source":"# Get file paths for train, validation and test\n\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\n\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')\n\nprint(\"# TRAINING_FILENAMES\", len(train_filenames))\nprint(\"# TEST_FILENAMES\", len(test_filenames))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:45:34.487278Z","iopub.execute_input":"2021-06-14T11:45:34.487606Z","iopub.status.idle":"2021-06-14T11:45:34.920241Z","shell.execute_reply.started":"2021-06-14T11:45:34.487577Z","shell.execute_reply":"2021-06-14T11:45:34.919442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Image loading functions\nWe create some functions for loading TFRecords to get the images, target and image names\n\n* **decode_image**: to transform images to a tensor, normalize it, and reshape it into the correct shape for TPU\n* **read_tfrecord**: read the TFRecord, returns the image tensor, and based on the input arguments, return the label value, image name, or nothing (0)\n* **load_dataset**: reads data from the TFRecords. Here we can choose whether to shuffle the data or not. We will do that for the train, but not the validation and test dataset.\n* **count_data_items**: counts the number of images in a file\n* **data_augment**: performs data augmentation; techniques used to increase the amount of data by adding slightly modified copies of already existing data or newly created synthetic data from existing data\n* **plot_transform**: plots some examples of augmented images","metadata":{}},{"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=512):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:45:46.299465Z","iopub.execute_input":"2021-06-14T11:45:46.300078Z","iopub.status.idle":"2021-06-14T11:45:46.314187Z","shell.execute_reply.started":"2021-06-14T11:45:46.300040Z","shell.execute_reply":"2021-06-14T11:45:46.313282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    # decode a JPEG-encoded image to a uint8 tensor\n    image = tf.image.decode_jpeg(image, channels=3) \n    # cast tensor to float32 and normalize to [0, 1] range\n    image = tf.cast(image, tf.float32)/255.0 \n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMG_SIZES[0:2], 3]) \n    return image\n\ndef read_tfrecord(example, labeled, return_imgname=False):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    # returns a dataset of (image, label) pairs if labeled=True\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    # returns a dataset of (image, image_name) pairs if return_imgname=True\n    if return_imgname:\n        return image, idnum\n    # else returns a dataset of (image, 0) pairs \n    return image, 0\n\ndef load_dataset(filenames, labeled=True, ordered=False, return_imgname=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order) \n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled, \n                                  return_imgname=return_imgname), num_parallel_calls=AUTO)\n    , or (image, id) pairs if labeled=False\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef data_augment(image, label=None, seed=SEED):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement when \n    # loading dataset (below cell), this happens essentially for free on TPU. Data pipeline\n    # code is executed on the \"CPU\" part of the TPU while the TPU itself is \n    # computing gradients.\n    image = transform(image, IMG_SIZES[0])\n    #image = tf.image.rot90(image,k=np.random.randint(4)) # rotate\n    image = tf.image.random_flip_left_right(image, seed=seed) # flip horizontal\n    image = tf.image.random_flip_up_down(image, seed=seed) # flip vertical\n    image = tf.image.random_brightness(image, max_delta=0.2) # random brightness\n    image = tf.image.random_contrast(image, 0.8, 1.2) # random contrast\n    image = tf.image.random_saturation(image, 0.7, 1.3) # random saturation\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\n# plot augmented images sample\ndef plot_transform(num_images):\n    fig, ax = plt.subplots(nrows=3, ncols=num_images, figsize=(12,5))\n    x = (load_dataset(train_filenames, labeled=True)\n                     .shuffle(SEED)\n                     .batch(BATCH_SIZES[0],drop_remainder=True)                 \n                     .prefetch(AUTO)\n                     .unbatch().take(5))\n    images = []\n    imgs=[]\n    for r in range(3):\n        image,_ = iter(x).next()\n        images.append(image)\n        for i in range(0,num_images):\n            image = data_augment(image=images[r])\n            imgs.append(image)\n    for img in range(len(imgs)):          \n        ax[img//num_images][img%num_images].imshow(imgs[img])\n        ax[img//num_images][img%num_images].axis('off') ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:45:58.731222Z","iopub.execute_input":"2021-06-14T11:45:58.731824Z","iopub.status.idle":"2021-06-14T11:45:58.754486Z","shell.execute_reply.started":"2021-06-14T11:45:58.731783Z","shell.execute_reply":"2021-06-14T11:45:58.753417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function for plotting images in grid\ndef show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target_or_imgid = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \neg_ds = (load_dataset(train_filenames, labeled=True)\n                     .batch(BATCH_SIZES[0],drop_remainder=True)                 \n                     .prefetch(AUTO)\n                     .unbatch().take(10*6))  \n\nshow_dataset(64, 10, 6, eg_ds) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:46:08.403943Z","iopub.execute_input":"2021-06-14T11:46:08.404282Z","iopub.status.idle":"2021-06-14T11:46:09.630442Z","shell.execute_reply.started":"2021-06-14T11:46:08.404254Z","shell.execute_reply":"2021-06-14T11:46:09.629389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Image augmentation examples. ","metadata":{}},{"cell_type":"code","source":"plot_transform(7) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:46:19.378018Z","iopub.execute_input":"2021-06-14T11:46:19.378398Z","iopub.status.idle":"2021-06-14T11:46:24.672230Z","shell.execute_reply.started":"2021-06-14T11:46:19.378363Z","shell.execute_reply":"2021-06-14T11:46:24.671277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of Train Files\", count_data_items(train_filenames))\nprint(\"Number of Test Files\", count_data_items(test_filenames))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:53:56.854383Z","iopub.execute_input":"2021-06-14T11:53:56.854822Z","iopub.status.idle":"2021-06-14T11:53:56.861990Z","shell.execute_reply.started":"2021-06-14T11:53:56.854751Z","shell.execute_reply":"2021-06-14T11:53:56.860703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Transfer Learning EffNet (CNN)","metadata":{}},{"cell_type":"markdown","source":"#### Learning Rate Train Schedule\nThis is a common train schedule for transfer learning. The learning rate starts near zero, then increases to a maximum, then decays over time. ","metadata":{}},{"cell_type":"code","source":"def lrfn(epoch):\n    if epoch < lr_ramp_ep:\n        lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n    elif epoch < lr_ramp_ep + lr_sus_ep:\n        lr = lr_max\n\n    else:\n        lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n\n    return lr\n\ndef get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.000003 * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.3\n       \n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:46:49.650135Z","iopub.execute_input":"2021-06-14T11:46:49.650468Z","iopub.status.idle":"2021-06-14T11:46:49.657191Z","shell.execute_reply.started":"2021-06-14T11:46:49.650439Z","shell.execute_reply":"2021-06-14T11:46:49.655959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_start   = 0.000005\nlr_max     = 0.000003 * BATCH_SIZES[0]\nlr_min     = 0.000001\nlr_ramp_ep = 5\nlr_sus_ep  = 0\nlr_decay   = 0.3","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:46:55.476000Z","iopub.execute_input":"2021-06-14T11:46:55.476360Z","iopub.status.idle":"2021-06-14T11:46:55.482055Z","shell.execute_reply.started":"2021-06-14T11:46:55.476323Z","shell.execute_reply":"2021-06-14T11:46:55.481204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ReshapeLayer(x):\n    '''\n    Reshape CNN output\n    '''\n    shape = x.shape \n    # H,W * channel\n    reshape = Reshape((shape[1],1))(x)\n\n    return reshape","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:47:08.095002Z","iopub.execute_input":"2021-06-14T11:47:08.095328Z","iopub.status.idle":"2021-06-14T11:47:08.099993Z","shell.execute_reply.started":"2021-06-14T11:47:08.095299Z","shell.execute_reply":"2021-06-14T11:47:08.098738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_training_curves(history, name, model_name):\n    '''\n    Plots the training process\n    '''\n    fig, (ax1, ax2) = plt.subplots(2, figsize=(20,15))\n    fig.suptitle(model_name, size=20)\n    \n    # plot AUC\n    ax1.plot(np.arange(EPOCHS),history.history['auc'],'-o',\n             label='Train AUC',color='#ff7f0e')\n    ax1.plot(np.arange(EPOCHS),history.history['val_auc'],'-o',\n             label='Val AUC',color='#1f77b4')\n    x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])\n    xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n    ax1.scatter(x,y,s=200,color='#1f77b4')\n    ax1.text(x-0.03*xdist,y-0.05*ydist,'max auc\\n%.2f'%y,size=14)\n    ax1.set_ylabel('AUC',size=14); ax1.set_xlabel('Epoch',size=14)\n    ax1.set_title('AUC')\n    ax1.set_xticks(list(range(EPOCHS)))\n    ax1.set_xticklabels(list(range(1, EPOCHS+1)))\n    ax1.legend(loc=2)\n\n    #  plot loss\n    ax2.plot(np.arange(EPOCHS),history.history['loss'],'-o',\n              label='Train Loss',color='#2ca02c')\n    ax2.plot(np.arange(EPOCHS),history.history['val_loss'],'-o',\n              label='Val Loss',color='#d62728')\n    x = np.argmin(history.history['val_loss'] )\n    y = np.min(history.history['val_loss'] )\n    ydist = plt.ylim()[1] - plt.ylim()[0]\n    ax2.scatter(x,y,s=200,color='#d62728')\n    ax2.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n    ax2.set_ylabel('Loss',size=14); ax2.set_xlabel('Epoch',size=14)\n    ax2.set_title('Loss')\n    ax2.set_xticks(list(range(EPOCHS)))\n    ax2.set_xticklabels(list(range(1, EPOCHS+1)))\n    ax2.legend(loc=3)\n    plt.savefig(name + '.png')\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:47:19.767213Z","iopub.execute_input":"2021-06-14T11:47:19.767576Z","iopub.status.idle":"2021-06-14T11:47:19.778999Z","shell.execute_reply.started":"2021-06-14T11:47:19.767544Z","shell.execute_reply":"2021-06-14T11:47:19.778264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get and load train, validation and test dataset","metadata":{}},{"cell_type":"markdown","source":"Split files to get train and validation filenames","metadata":{}},{"cell_type":"code","source":"files_train, files_valid = train_test_split(\n    train_filenames, test_size = 0.20, random_state = SEED)\n\n# shuffle training set\nnp.random.shuffle(files_train) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:47:28.115328Z","iopub.execute_input":"2021-06-14T11:47:28.115812Z","iopub.status.idle":"2021-06-14T11:47:28.120849Z","shell.execute_reply.started":"2021-06-14T11:47:28.115765Z","shell.execute_reply":"2021-06-14T11:47:28.119867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of Train Files after Splitting\", count_data_items(files_train))\nprint(\"Number of Validation Files\", count_data_items(files_valid))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:54:17.375028Z","iopub.execute_input":"2021-06-14T11:54:17.375392Z","iopub.status.idle":"2021-06-14T11:54:17.381290Z","shell.execute_reply.started":"2021-06-14T11:54:17.375360Z","shell.execute_reply":"2021-06-14T11:54:17.380265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get test set filenames","metadata":{}},{"cell_type":"code","source":"files_test = np.sort(\n    np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:49:02.259029Z","iopub.execute_input":"2021-06-14T11:49:02.259411Z","iopub.status.idle":"2021-06-14T11:49:02.334869Z","shell.execute_reply.started":"2021-06-14T11:49:02.259378Z","shell.execute_reply":"2021-06-14T11:49:02.333857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load train, validation and test dataset","metadata":{}},{"cell_type":"code","source":"train_dataset = (load_dataset(files_train, labeled=True)\n                     .repeat() # repeat to continue getting data for aug\n                     .map(data_augment, num_parallel_calls=AUTO) # data augmentation\n                     .shuffle(SEED)\n                     .batch(BATCH_SIZES[0],drop_remainder=True)\n                     # prefetch next batch while training (autotune prefetch buffer size)\n                     .prefetch(AUTO)) \n\nds_valid = (load_dataset(files_valid, labeled=True, ordered=True)                                        \n                     .cache()     \n                     .repeat()   # repeat for data aug during val\n                     .map(data_augment, num_parallel_calls=AUTO)  # data augmentation \n                     .batch(BATCH_SIZES[0]*4)  # X4 to speed up training\n                     .prefetch(AUTO))\n\nds_test = (load_dataset(files_test, labeled=False, ordered=True) # do not shuffle\n                     .repeat()                                   # repeat for TTA\n                     .map(data_augment, num_parallel_calls=AUTO) # data augmentation \n                     .batch(BATCH_SIZES[0]*4)\n                     .prefetch(AUTO))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:55:17.499083Z","iopub.execute_input":"2021-06-14T12:55:17.499451Z","iopub.status.idle":"2021-06-14T12:55:17.967712Z","shell.execute_reply.started":"2021-06-14T12:55:17.499414Z","shell.execute_reply":"2021-06-14T12:55:17.966709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train and Evaluate model","metadata":{}},{"cell_type":"code","source":"# initialize storage\npred_ = []; tar_ = []; val_ = []; names_ = []\n# initialize for prediction storage\npreds = np.zeros((count_data_items(test_filenames),1))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:32:46.502303Z","iopub.execute_input":"2021-06-14T12:32:46.502660Z","iopub.status.idle":"2021-06-14T12:32:46.507286Z","shell.execute_reply.started":"2021-06-14T12:32:46.502630Z","shell.execute_reply":"2021-06-14T12:32:46.506596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2\n\nif tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    \nK.clear_session()\nwith strategy.scope():\n    model = tf.keras.Sequential([\n        efn.EfficientNetB6(input_shape=(IMG_SIZES[0],IMG_SIZES[0], 3),\n                           weights='imagenet',include_top=False),\n        GlobalAveragePooling2D(),\n        # add fully connected layer, with sigmoid activation since only 2 categories\n        Dense(1, activation='sigmoid') \n    ])\n    \n    model.compile(\n        optimizer='adam',\n        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=[tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n                 tf.keras.metrics.AUC(name='auc')])\n    print(model.summary())\n\n# for saving best model from the best epoch \nsv = tf.keras.callbacks.ModelCheckpoint(\n        'cnn_best.h5', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n\nprint('#'*25)\nprint('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n      (IMG_SIZES[0],EFF_NETS,BATCH_SIZES[0]))\n\n# TRAIN\nprint('Training...')         \nhistory = model.fit(\n    train_dataset, \n    epochs=EPOCHS, \n    callbacks=[sv,get_lr_callback(BATCH_SIZES[0])],     # lr schedule\n    steps_per_epoch=count_data_items(files_train) // BATCH_SIZES[0],\n    validation_data=load_dataset(files_valid, labeled=True)                                        \n                     .cache()\n                     .batch(BATCH_SIZES[0])\n                     .prefetch(AUTO),                         \n    verbose=VERBOSE) \n\n# LOAD BEST MODEL\nprint('Loading best model...')\nmodel.load_weights('cnn_best.h5')    ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:34:43.505108Z","iopub.execute_input":"2021-06-14T12:34:43.505513Z","iopub.status.idle":"2021-06-14T12:53:33.392244Z","shell.execute_reply.started":"2021-06-14T12:34:43.505478Z","shell.execute_reply":"2021-06-14T12:53:33.390947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict on Validation and test set using test time augmentation","metadata":{}},{"cell_type":"code","source":"print('Predicting Valid with TTA...')\n\nct_valid = count_data_items(files_valid)\nSTEPS = TTA * ct_valid/BATCH_SIZES[0]/4  # number of steps to go through all TTA images\n# slice to throw away images that pass the steps\npred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n# store the average of each valid image \npred_.append(np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1)) \n\n# GET OOF TARGETS, FOLDS, AND NAMES\n# get targets \n# do not repeat=True here as we only want the target values \nds_valid = (load_dataset(files_valid, labeled=True, ordered=True) # do not shuffle\n                    .cache()\n                    .batch(BATCH_SIZES[0]*4)\n                    .prefetch(AUTO))\ntar_.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) ) \n# get names\nds = (load_dataset(files_valid, labeled=False, return_imgname=True, ordered=True)                                        \n            .cache()     \n            .batch(BATCH_SIZES[0]*4)                   \n            .prefetch(AUTO)) \nnames_.append(np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(\n    ds.unbatch())]))      \n\n# PREDICT TEST USING TTA\nprint('Predicting Test with TTA...')\nct_test = count_data_items(files_test)\nSTEPS = TTA * ct_test/BATCH_SIZES[0]/4 # number of steps to go through all TTA images\n# slice to throw away images that pass the steps\npred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n# store the average pred of each test image\npreds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1)\n\n# REPORT RESULTS\nauc_ = roc_auc_score(tar_[-1],pred_[-1])\nval_.append(np.max(history.history['val_auc']))\nprint('#### AUC without TTA = %.3f, with TTA = %.3f'%(val_[-1],auc_)) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:55:32.097057Z","iopub.execute_input":"2021-06-14T12:55:32.097442Z","iopub.status.idle":"2021-06-14T12:58:32.603637Z","shell.execute_reply.started":"2021-06-14T12:55:32.097410Z","shell.execute_reply":"2021-06-14T12:58:32.602418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(history, 'effnet', 'EfficientNet B6')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE TO DISK\ndf = pd.DataFrame(dict(\n    image_name = names_[-1], target=tar_[-1], pred = pred_[-1]))\ndf.to_csv('effnet.csv',index=False) \ndf.head() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:02:12.916706Z","iopub.execute_input":"2021-06-14T13:02:12.917067Z","iopub.status.idle":"2021-06-14T13:02:12.946448Z","shell.execute_reply.started":"2021-06-14T13:02:12.917036Z","shell.execute_reply":"2021-06-14T13:02:12.945518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ROC and PR curves","metadata":{}},{"cell_type":"code","source":"# AUROC\nfpr, tpr, _ = roc_curve(df.target, df.pred)\nroc_auc = auc(fpr, tpr)\n\n# AUPRC\nprecision, recall, thresholds = precision_recall_curve(df.target, df.pred)\naverage_precision = average_precision_score(df.target, df.pred)\n\n# PLOT\n# auroc\nfig = plt.figure(figsize=(18,6))\nax1 = fig.add_subplot(121)\n\nfig.suptitle('ROC and PRC Curves for EfficientNet B6', size=25)\n\nax1.plot([0, 1], [0, 1], linestyle='--', lw=4, color='r',\n        label='Chance', alpha=.8)\n\nax1.plot(fpr, tpr, color='b',\n        label=r'ROC (AUC = %0.2f)' % (roc_auc),\n        lw=4, alpha=.8)    \n    \nax1.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])\nax1.set_title(\"Receiver Operating Characteristic Curve\", size=20)\nax1.set_xlabel('False Positive Rate',size=20); plt.xticks(size=15)\nax1.set_ylabel('True Positive Rate',size=20); plt.yticks(size=15)\nax1.legend(loc=\"lower right\",prop={\"size\":15})\n\n# auprc\nax2 = fig.add_subplot(122)\nax2.step(recall, precision, where='post', color='b',\n        label=r'AP (AP = %0.2f)' % (average_precision),\n        lw=4, alpha=.8)    \n    \nax2.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])\nax2.set_title(\"Precision Recall Curve\", size=20)\nax2.set_xlabel('Recall',size=20); plt.xticks(size=15)\nax2.set_ylabel('Precision',size=20); plt.yticks(size=15)\nax2.legend(loc=\"lower left\",prop={\"size\":15})\nplt.savefig('effnet_roc.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:02:20.840013Z","iopub.execute_input":"2021-06-14T13:02:20.840370Z","iopub.status.idle":"2021-06-14T13:02:21.417007Z","shell.execute_reply.started":"2021-06-14T13:02:20.840340Z","shell.execute_reply":"2021-06-14T13:02:21.415991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to calculate the F1 Score\ndef calc_f1(prec, recall):\n    return 2*(prec*recall)/(prec+recall) if recall and prec else 0","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:01:01.137717Z","iopub.execute_input":"2021-06-14T13:01:01.138098Z","iopub.status.idle":"2021-06-14T13:01:01.141979Z","shell.execute_reply.started":"2021-06-14T13:01:01.138066Z","shell.execute_reply":"2021-06-14T13:01:01.141218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the f1 score for each threshold\nf1score = [calc_f1(precision[i], recall[i]) for i in range(len(thresholds))]\n\n# Get the highest f1score\nidx = np.argmax(f1score)\n\n# Get the highest precision, recall, threshold and f1score\nprecision = round(precision[idx], 4)\nrecall = round(recall[idx], 4)\nthreshold = round(thresholds[idx], 4)\nf1score = round(f1score[idx], 4)\n\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('Threshold:', threshold)\nprint('F1 Score:', f1score)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:02:44.345371Z","iopub.execute_input":"2021-06-14T13:02:44.345722Z","iopub.status.idle":"2021-06-14T13:02:44.360037Z","shell.execute_reply.started":"2021-06-14T13:02:44.345686Z","shell.execute_reply":"2021-06-14T13:02:44.358838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a confusion matrix\nbinary_preds = [0 if x < threshold else 1 for x in df.pred]\ncm = confusion_matrix(df.target, binary_preds)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True)\nplt.xlabel('Predicted label', size=10)\nplt.ylabel('True label', size=10)\nplt.title('EfficientNet B6 + BiLSTM Confusion Matrix', size=15)\nplt.savefig('effnet_cm.png')\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:03:59.196509Z","iopub.execute_input":"2021-06-14T13:03:59.196899Z","iopub.status.idle":"2021-06-14T13:03:59.531734Z","shell.execute_reply.started":"2021-06-14T13:03:59.196858Z","shell.execute_reply":"2021-06-14T13:03:59.530573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Predict for an image","metadata":{}},{"cell_type":"code","source":"IMAGE_PATH = \"../input/siim-isic-melanoma-classification/jpeg/test/ISIC_0052060.jpg\"\nimg = tf.keras.preprocessing.image.load_img(IMAGE_PATH, \n                                            target_size=(IMG_SIZES[0], IMG_SIZES[0]))\nplt.imshow(img)\norigin_img = img","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:05:36.987662Z","iopub.execute_input":"2021-06-14T13:05:36.988034Z","iopub.status.idle":"2021-06-14T13:05:37.568930Z","shell.execute_reply.started":"2021-06-14T13:05:36.988000Z","shell.execute_reply":"2021-06-14T13:05:37.567982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the prediction for the image\nprediction = model.predict(np.expand_dims(img, axis=0))\nbinary_prediction = [0 if x < 0.5 else 1 for x in prediction]\nprint(\"Prediction: \" + (\"Benign\" if binary_prediction == 0 else \"Malignant\")) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T13:06:18.287623Z","iopub.execute_input":"2021-06-14T13:06:18.288003Z","iopub.status.idle":"2021-06-14T13:06:31.759023Z","shell.execute_reply.started":"2021-06-14T13:06:18.287969Z","shell.execute_reply":"2021-06-14T13:06:31.757714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. EffNet + BiLSTM (Hybrid)","metadata":{}},{"cell_type":"markdown","source":"#### Build, Train and Evaluate model  ","metadata":{}},{"cell_type":"code","source":"# initialize storage\npred_ = []; tar_ = []; val_ = []; names_ = []\n# initialize for prediction storage\npreds = np.zeros((count_data_items(test_filenames),1))  ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:54:29.951166Z","iopub.execute_input":"2021-06-14T11:54:29.951525Z","iopub.status.idle":"2021-06-14T11:54:29.956405Z","shell.execute_reply.started":"2021-06-14T11:54:29.951493Z","shell.execute_reply":"2021-06-14T11:54:29.955304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2\n\nif tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    \nK.clear_session()\nwith strategy.scope():\n    model = tf.keras.Sequential([\n        efn.EfficientNetB6(input_shape=(IMG_SIZES[0],IMG_SIZES[0], 3),\n                           weights='imagenet',include_top=False),\n        GlobalAveragePooling2D(),\n        # reshape layer\n        Lambda(ReshapeLayer),\n        # add BiLSTM layer\n        Bidirectional(LSTM(150, return_sequences=True, dropout=0.3)),\n        Bidirectional(LSTM(96, dropout=0.3)),\n        # dense layer\n        Dense(8, activation='relu'),\n        # add fully connected layer, with sigmoid activation since only 2 categories\n        Dense(1, activation='sigmoid') \n    ])\n    \n    model.compile(\n        optimizer='adam',\n        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=[tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n                 tf.keras.metrics.AUC(name='auc')])\n    print(model.summary())\n\n# for saving best model from the best epoch \nsv = tf.keras.callbacks.ModelCheckpoint(\n        'bi_best.h5', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n\nprint('#'*25)\nprint('#### Image Size %i with EfficientNet B%i + BiLSTM and batch_size %i'%\n      (IMG_SIZES[0],EFF_NETS,BATCH_SIZES[0]))\n\n# TRAIN\nprint('Training...')         \nhistory = model.fit(\n    train_dataset, \n    epochs=EPOCHS, \n    callbacks=[sv,get_lr_callback(BATCH_SIZES[0])],     # lr schedule\n    steps_per_epoch=count_data_items(files_train) // BATCH_SIZES[0],\n    validation_data=load_dataset(files_valid, labeled=True)                                        \n                     .cache()\n                     .batch(BATCH_SIZES[0])\n                     .prefetch(AUTO),                         \n    verbose=VERBOSE) \n\n# LOAD BEST MODEL\nprint('Loading best model...')\nmodel.load_weights('bi_best.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:54:39.479966Z","iopub.execute_input":"2021-06-14T11:54:39.480307Z","iopub.status.idle":"2021-06-14T12:19:02.509101Z","shell.execute_reply.started":"2021-06-14T11:54:39.480276Z","shell.execute_reply":"2021-06-14T12:19:02.507906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict valid and test dataset using test time augmentation (TTA). ","metadata":{}},{"cell_type":"code","source":"print('Predicting Valid with TTA...')\n\nct_valid = count_data_items(files_valid)\nSTEPS = TTA * ct_valid/BATCH_SIZES[0]/4  # number of steps to go through all TTA images\n# slice to throw away images that pass the steps\npred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n# store the average of each valid image \npred_.append(np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1)) \n\n# GET OOF TARGETS, FOLDS, AND NAMES\n# get targets \n# do not repeat=True here as we only want the target values \nds_valid = (load_dataset(files_valid, labeled=True, ordered=True) # do not shuffle\n                    .cache()\n                    .batch(BATCH_SIZES[0]*4)\n                    .prefetch(AUTO))\ntar_.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) ) \n# get names\nds = (load_dataset(files_valid, labeled=False, return_imgname=True, ordered=True)                                        \n            .cache()     \n            .batch(BATCH_SIZES[0]*4)                   \n            .prefetch(AUTO)) \nnames_.append(np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(\n    ds.unbatch())]))      \n\n# PREDICT TEST USING TTA\nprint('Predicting Test with TTA...')\nct_test = count_data_items(files_test)\nSTEPS = TTA * ct_test/BATCH_SIZES[0]/4 # number of steps to go through all TTA images\n# slice to throw away images that pass the steps\npred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n# store the average pred of each test image\npreds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1)\n\n# REPORT RESULTS\nauc_ = roc_auc_score(tar_[-1],pred_[-1])\nval_.append(np.max(history.history['val_auc']))\nprint('#### AUC without TTA = %.3f, with TTA = %.3f'%(val_[-1],auc_)) ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:19:26.482230Z","iopub.execute_input":"2021-06-14T12:19:26.482625Z","iopub.status.idle":"2021-06-14T12:25:02.131331Z","shell.execute_reply.started":"2021-06-14T12:19:26.482587Z","shell.execute_reply":"2021-06-14T12:25:02.130277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(history, 'bi_lstm', 'EfficientNet B6 + BiLSTM')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:25:58.748051Z","iopub.execute_input":"2021-06-14T12:25:58.748388Z","iopub.status.idle":"2021-06-14T12:25:59.477510Z","shell.execute_reply.started":"2021-06-14T12:25:58.748357Z","shell.execute_reply":"2021-06-14T12:25:59.476383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE TO DISK\ndf = pd.DataFrame(dict(\n    image_name = names_[-1], target=tar_[-1], pred = pred_[-1]))\ndf.to_csv('bilstm.csv',index=False) \ndf.head() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:26:17.081166Z","iopub.execute_input":"2021-06-14T12:26:17.081638Z","iopub.status.idle":"2021-06-14T12:26:17.113106Z","shell.execute_reply.started":"2021-06-14T12:26:17.081603Z","shell.execute_reply":"2021-06-14T12:26:17.112212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### ROC and PR Curves","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score\n\n# AUROC\nfpr, tpr, _ = roc_curve(df.target, df.pred)\nroc_auc = auc(fpr, tpr)\n\n# AUPRC\nprecision, recall, thresholds = precision_recall_curve(df.target, df.pred)\naverage_precision = average_precision_score(df.target, df.pred)\n\n# PLOT\n# auroc\nfig = plt.figure(figsize=(18,6))\nax1 = fig.add_subplot(121)\n\nfig.suptitle('ROC and PRC Curves for EfficientNet B6 + BiLSTM', size=25)\n\nax1.plot([0, 1], [0, 1], linestyle='--', lw=4, color='r',\n        label='Chance', alpha=.8)\n\nax1.plot(fpr, tpr, color='b',\n        label=r'ROC (AUC = %0.2f)' % (roc_auc),\n        lw=4, alpha=.8)    \n    \nax1.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])\nax1.set_title(\"Receiver Operating Characteristic Curve\", size=20)\nax1.set_xlabel('False Positive Rate',size=20); plt.xticks(size=15)\nax1.set_ylabel('True Positive Rate',size=20); plt.yticks(size=15)\nax1.legend(loc=\"lower right\",prop={\"size\":15})\n\n# auprc\nax2 = fig.add_subplot(122)\nax2.step(recall, precision, where='post', color='b',\n        label=r'AP (AP = %0.2f)' % (average_precision),\n        lw=4, alpha=.8)    \n    \nax2.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05])\nax2.set_title(\"Precision Recall Curve\", size=20)\nax2.set_xlabel('Recall',size=20); plt.xticks(size=15)\nax2.set_ylabel('Precision',size=20); plt.yticks(size=15)\nax2.legend(loc=\"lower left\",prop={\"size\":15})\nplt.savefig('bilstm_roc.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:29:11.288911Z","iopub.execute_input":"2021-06-14T12:29:11.289499Z","iopub.status.idle":"2021-06-14T12:29:11.842072Z","shell.execute_reply.started":"2021-06-14T12:29:11.289460Z","shell.execute_reply":"2021-06-14T12:29:11.841194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to calculate the F1 Score\ndef calc_f1(prec, recall):\n    return 2*(prec*recall)/(prec+recall) if recall and prec else 0","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:29:31.488661Z","iopub.execute_input":"2021-06-14T12:29:31.489140Z","iopub.status.idle":"2021-06-14T12:29:31.493444Z","shell.execute_reply.started":"2021-06-14T12:29:31.489108Z","shell.execute_reply":"2021-06-14T12:29:31.492388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the f1 score for each threshold\nf1score = [calc_f1(precision[i], recall[i]) for i in range(len(thresholds))]\n\n# Get the highest f1score\nidx = np.argmax(f1score)\n\n# Get the highest precision, recall, threshold and f1score\nprecision = round(precision[idx], 4)\nrecall = round(recall[idx], 4)\nthreshold = round(thresholds[idx], 4)\nf1score = round(f1score[idx], 4)\n\nprint('Precision:', precision)\nprint('Recall:', recall)\nprint('Threshold:', threshold)\nprint('F1 Score:', f1score)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T12:29:41.155718Z","iopub.execute_input":"2021-06-14T12:29:41.156097Z","iopub.status.idle":"2021-06-14T12:29:41.170443Z","shell.execute_reply.started":"2021-06-14T12:29:41.156063Z","shell.execute_reply":"2021-06-14T12:29:41.169598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a confusion matrix\nbinary_preds = [0 if x < threshold else 1 for x in df.pred]\ncm = confusion_matrix(df.target, binary_preds)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True)\nplt.xlabel('Predicted label', size=10)\nplt.ylabel('True label', size=10)\nplt.title('EfficientNet B6 + BiLSTM Confusion Matrix', size=15)\nplt.savefig('bilstm_cm.png')\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:26:16.710042Z","iopub.execute_input":"2021-06-14T11:26:16.710469Z","iopub.status.idle":"2021-06-14T11:26:17.079411Z","shell.execute_reply.started":"2021-06-14T11:26:16.710429Z","shell.execute_reply":"2021-06-14T11:26:17.078082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Predict for an image","metadata":{}},{"cell_type":"code","source":"IMAGE_PATH = \"../input/siim-isic-melanoma-classification/jpeg/test/ISIC_0052060.jpg\"\nimg = tf.keras.preprocessing.image.load_img(IMAGE_PATH, \n                                            target_size=(IMG_SIZES[0], IMG_SIZES[0]))\nplt.imshow(img)\norigin_img = img ","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:40:25.509901Z","iopub.execute_input":"2021-06-14T11:40:25.510261Z","iopub.status.idle":"2021-06-14T11:40:26.127633Z","shell.execute_reply.started":"2021-06-14T11:40:25.510230Z","shell.execute_reply":"2021-06-14T11:40:26.126588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the prediction for the image\nprediction = model.predict(np.expand_dims(img, axis=0))\nbinary_prediction = [0 if x < 0.5 else 1 for x in prediction]\nprint(\"Prediction: \" + (\"Benign\" if binary_prediction == 0 else \"Malignant\"))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T11:41:18.138087Z","iopub.execute_input":"2021-06-14T11:41:18.138592Z","iopub.status.idle":"2021-06-14T11:41:33.494405Z","shell.execute_reply.started":"2021-06-14T11:41:18.138555Z","shell.execute_reply":"2021-06-14T11:41:33.493746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}