{"cells":[{"metadata":{},"cell_type":"markdown","source":"> # Overview:\n\n* Efficientnet B6 trained on [512x521 External Dataset](https://www.kaggle.com/cdeotte/512x512-melanoma-tfrecords-70k-images). \n* Uses get_cosine_schedule_with_warmup as a scheduler with a warmup of 5 Epochs.\n* Test Time Augmentation(TTA) of 4.\n* BCE loss with label smoothing of 0.05\n\n> ### Credits:\n\n* I would Like to thank, [Chris Deotte](https://www.kaggle.com/cdeotte) for the dataset.\n* I have used most of the TPU helper function from [Wei Hao Khoong](https://www.kaggle.com/khoongweihao)'s Multiple Model Training [notebook](https://www.kaggle.com/khoongweihao/siim-isic-multiple-model-training-stacking). Thank you.\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport cv2\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # knockknock\n\n* I used [knockknock](https://github.com/huggingface/knockknock) from huggingface to send the model training status to my telegram bot. It ll alert the user when the training is complete. \n* Go through the [readme](https://github.com/huggingface/knockknock#telegram) to create a telegram bot. After successfully creating the telegram bot the ` token ` and ` chat_id ` has to be added as a secret to kaggle notebook.\n* If you don't want to use knockknock, set ` use_knockknock = False `","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"use_knockknock = True #set this to false if you are not using knockknock.\nif use_knockknock:\n    !pip install knockknock\n    from knockknock import telegram_sender \n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    token = user_secrets.get_secret(\"token\") #kaggle secret token\n    chat_id = user_secrets.get_secret(\"chat_id\") #kaggle secret chat_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> #  TPU configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nDATASET = '512x512-melanoma-tfrecords-70k-images'\nGCS_PATH = KaggleDatasets().get_gcs_path(DATASET)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Hyperparameter tuning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nSIZE = [512,512]\nLR = 0.00004\nEPOCHS = 12\nWARMUP = 5\nWEIGHT_DECAY = 0\nLABEL_SMOOTHING = 0.05\nTTA = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.2,random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE, 3])\n    return image\n\ndef data_augment(image, label=None, seed=SEED):\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string), }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    return image, image_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n              .with_options(ignore_order)\n              .map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO))\n            \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef plot_transform(num_images):\n    plt.figure(figsize=(30,10))\n    x = load_dataset(train_filenames, labeled=False)\n    image,_ = iter(x).next()\n    for i in range(1,num_images+1):\n        plt.subplot(1,num_images+1,i)\n        plt.axis('off')\n        image = data_augment(image=image)\n        plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Visualizing Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_transform(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (load_dataset(train_filenames, labeled=True)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (load_dataset(valid_filenames, labeled=True)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.Sequential([\n        efn.EfficientNetB6(input_shape=(*SIZE, 3),weights='imagenet',pooling='avg',include_top=False),\n        Dense(1, activation='sigmoid')\n    ])\n        \n    model.compile(\n        optimizer='adam',\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=['accuracy',tf.keras.metrics.AUC(name='auc')])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Scheduler\n\n*  Modified version of the [get_cosine_schedule_with_warmup](https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup) from huggingface.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n    \"\"\"\n    Modified version of the get_cosine_schedule_with_warmup from huggingface.\n    (https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)\n\n    Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_knockknock == True:\n    @telegram_sender(token=token, chat_id=int(chat_id))\n    def train():\n        STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\n        history = model.fit(\n            train_dataset, \n            epochs=EPOCHS, \n            callbacks=[lr_schedule],\n            steps_per_epoch=STEPS_PER_EPOCH,\n            validation_data=valid_dataset)\n\n        string = 'Train acc:{:.4f} Train loss:{:.4f} AUC: {:.4f}, Val acc:{:.4f} Val loss:{:.4f} Val AUC: {:.4f}'.format( \\\n            model.history.history['accuracy'][-1],model.history.history['loss'][-1],\\\n            model.history.history['auc'][-1],\\\n            model.history.history['val_accuracy'][-1],model.history.history['val_loss'][-1],\\\n            model.history.history['val_auc'][-1])\n\n        return string\nelse:\n    def train():\n        STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\n        history = model.fit(\n            train_dataset, \n            epochs=EPOCHS, \n            callbacks=[lr_schedule],\n            steps_per_epoch=STEPS_PER_EPOCH,\n            validation_data=valid_dataset)\n        \n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Plotting training loss, accuracy and roc","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(20,15), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(\n    model.history.history['loss'], \n    model.history.history['val_loss'], \n    'loss', 311)\ndisplay_training_curves(\n    model.history.history['accuracy'], \n    model.history.history['val_accuracy'], \n    'accuracy', 312)\ndisplay_training_curves(\n    model.history.history['auc'], \n    model.history.history['val_auc'], \n    'accuracy', 313)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Prediction with TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames)\nsubmission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames, labeled=False,ordered=True)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(lambda image, image_name: image)\n    test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    test_pred = model.predict(test_dataset_images, verbose=1)\n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})\n    temp = submission_df.copy()\n    del temp['target']\n    submission_df['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)\npd.Series(np.round(submission_df['target'].values)).value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* I just printed the value counts to know the bifurcation between malignant and benign. It sometimes helps is deciding whether I should click submit or not.\n  (Note: It varies if we change the label smoothing)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}