{"cells":[{"metadata":{},"cell_type":"markdown","source":"Reference Code for this notebook\nhttps://www.kaggle.com/jagadish13/melanoma-detection-efficientnetb7-tpu-eda","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Please view Version 5 for the best results from this notebook","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Importing Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\nimport numpy as np \nimport pandas as pd \nimport re\nimport cv2\nimport math\nimport time\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Input, Concatenate, Flatten, Dense\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting up the TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry: ## Trying to check if a TPU cluster exists\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('TPU Master ', tpu.master)\nexcept ValueError:\n    tpu = None\n    \nif tpu: #In the case the cluster exists, we initialize it and connect to it and create a strategy for parallel processing\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint('Replicas', strategy.num_replicas_in_sync)    \n\nDATASET1 = 'melanoma-768x768'\nGCS_PATH1 = KaggleDatasets().get_gcs_path(DATASET1) #Getting the Google Cloud Storage path for the publically available dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Cleaning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_details = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntrain_details.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_details['sex'] = train_details['sex'].fillna('male')\ntrain_details['age_approx'] = train_details['age_approx'].fillna(train_details['age_approx'].mean())\ntrain_details['anatom_site_general_challenge'] = train_details['anatom_site_general_challenge'].fillna('head/neck')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nenc1 = LabelEncoder()\nenc2 = LabelEncoder()\n\ntrain_details['sex'] = enc1.fit_transform(train_details['sex'])\ntrain_details['anatom_site_general_challenge'] = enc2.fit_transform(train_details['anatom_site_general_challenge'])\n\nx_vec = train_details[['sex','age_approx','anatom_site_general_challenge']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Setting the Hyperparameters for the Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nSIZE1 = [768,768]\nLR = 0.00004\nEPOCHS = 12\nWARMUP = 5\nWEIGHT_DECAY = 0\nLABEL_SMOOTHING = 0.05\nTTA = 4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setting a Seed for Everything","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(SEED)\ntf.random.set_seed(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Getting the train and test file paths","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/train*.tfrec')\ntest_filenames1 = tf.io.gfile.glob(GCS_PATH1 + '/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_filenames1, valid_filenames1 = train_test_split(train_filenames1, test_size = 0.15, random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating some Helper Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels = 3)\n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE1, 3])\n    return image\n\ndef data_augment(image, label = None, seed = SEED):\n    image = tf.image.rot90(image, k = np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed = SEED)\n    image = tf.image.random_flip_up_down(image, seed = SEED)\n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef read_labeled_tfrecord(example):\n    LFormat = {'image': tf.io.FixedLenFeature([], tf.string),\n             'target': tf.io.FixedLenFeature([], tf.int64)}\n    example = tf.io.parse_single_example(example, LFormat)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    \n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UFormat = {'image': tf.io.FixedLenFeature([], tf.string),\n             'image_name': tf.io.FixedLenFeature([], tf.string)}\n    example = tf.io.parse_single_example(example, UFormat)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    \n    return image, image_name\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    ignore_order = tf.data.Options()\n    \n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO).with_options(ignore_order).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))\n    \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading the train and validation datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = (load_dataset(train_filenames1).map(data_augment, num_parallel_calls = AUTO).shuffle(SEED).batch(BATCH_SIZE, drop_remainder = True).repeat().prefetch(AUTO))\nvalid_dataset1 = (load_dataset(valid_filenames1, labeled=True).batch(BATCH_SIZE).repeat().prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implementing the Efficientnet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.Sequential([efn.EfficientNetB7(input_shape=(*SIZE1,3), weights='imagenet', include_top=False, pooling = 'avg'),\n            Dense(1, activation = 'sigmoid')])\n    \n    model.compile(optimizer='adam',loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=[tf.keras.metrics.AUC(name='auc')])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate Scheduling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"validationsize1 = count_data_items(valid_filenames1)\nprint(validationsize1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = (count_data_items(train_filenames1)) // BATCH_SIZE\n#class_weights = {0:0.5089, 1:28.3613}\nmodel.fit(train1, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset1,validation_steps=validationsize1//BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Testing with Test Time Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames1)\nsubmission_df1 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames1, labeled=False,ordered=True)\n    .map(data_augment, num_parallel_calls=AUTO)  \n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(lambda image, image_name: image)\n    test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    test_pred = model.predict(test_dataset_images, verbose=1) \n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})\n    temp = submission_df1.copy()   \n    del temp['target']  \n    submission_df1['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df1.to_csv('efficientnetb7_784.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Second Model for Ensemble","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DATASET2 = 'melanoma-512x512'\nGCS_PATH2 = KaggleDatasets().get_gcs_path(DATASET2) #Getting the Google Cloud Storage path for the publically available dataset\n\nSIZE2 = [512,512]\n\ntrain_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')\ntest_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/test*.tfrec')\n\ntrain_filenames2, valid_filenames2 = train_test_split(train_filenames2, test_size = 0.15, random_state = SEED)\n\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels = 3)\n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE2, 3])\n    return image\n\ndef data_augment(image, label = None, seed = SEED):\n    image = tf.image.rot90(image, k = np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed = SEED)\n    image = tf.image.random_flip_up_down(image, seed = SEED)\n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef read_labeled_tfrecord(example):\n    LFormat = {'image': tf.io.FixedLenFeature([], tf.string),\n             'target': tf.io.FixedLenFeature([], tf.int64)}\n    example = tf.io.parse_single_example(example, LFormat)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    \n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UFormat = {'image': tf.io.FixedLenFeature([], tf.string),\n             'image_name': tf.io.FixedLenFeature([], tf.string)}\n    example = tf.io.parse_single_example(example, UFormat)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    \n    return image, image_name\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    ignore_order = tf.data.Options()\n    \n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO).with_options(ignore_order).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))\n    \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ntrain2 = (load_dataset(train_filenames2).map(data_augment, num_parallel_calls = AUTO).shuffle(SEED).batch(BATCH_SIZE, drop_remainder = True).repeat().prefetch(AUTO))\nvalid_dataset2 = (load_dataset(valid_filenames2, labeled=True).batch(BATCH_SIZE).repeat().prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n\n    model2 = tf.keras.Sequential([\n        efn.EfficientNetB7(input_shape=(*SIZE2, 3),weights='imagenet',pooling='avg',include_top=False),\n        Dense(1, activation='sigmoid')\n    ])\n    \n    model2.compile(optimizer='adam',loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=[tf.keras.metrics.AUC(name='auc')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validationsize2 = count_data_items(valid_filenames2)\nprint(validationsize2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = (count_data_items(train_filenames2)) // BATCH_SIZE\n#class_weights = {0:0.5089, 1:28.3613}\nmodel2.fit(train2, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset2,validation_steps=validationsize2//BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames2)\nsubmission_df2 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames2, labeled=False,ordered=True)\n    .map(data_augment, num_parallel_calls=AUTO)  \n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(lambda image, image_name: image)\n    test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    test_pred = model2.predict(test_dataset_images, verbose=1) \n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})\n    temp = submission_df2.copy()   \n    del temp['target']  \n    submission_df2['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df2.to_csv('efficientnetb7_512.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}