{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport cv2\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> #  TPU configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nDATASET = '384x384-melanoma-tfrecords-70k-images'\nGCS_PATH = KaggleDatasets().get_gcs_path(DATASET)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Hyperparameter tuning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nSIZE = [384,384]\nLR = 0.00004\nEPOCHS = 15\nWARMUP = 5\nWEIGHT_DECAY = 0\nLABEL_SMOOTHING = 0.05\nTTA = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.2,random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE, 3])\n    return image\n\ndef data_augment(image, label=None, seed=SEED):\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string), }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    return image, image_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n              .with_options(ignore_order)\n              .map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO))\n            \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef plot_transform(num_images):\n    plt.figure(figsize=(30,10))\n    x = load_dataset(train_filenames, labeled=False)\n    image,_ = iter(x).next()\n    for i in range(1,num_images+1):\n        plt.subplot(1,num_images+1,i)\n        plt.axis('off')\n        image = data_augment(image=image)\n        plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Visualizing Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_transform(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (load_dataset(train_filenames, labeled=True)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (load_dataset(valid_filenames, labeled=True)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Model,loss function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_loss(alpha=0.25,gamma=2.0):\n    def focal_crossentropy(y_true, y_pred):\n        bce = K.binary_crossentropy(y_true, y_pred)\n        \n        y_pred = K.clip(y_pred, K.epsilon(), 1.- K.epsilon())\n        p_t = (y_true*y_pred) + ((1-y_true)*(1-y_pred))\n        \n        alpha_factor = 1\n        modulating_factor = 1\n\n        alpha_factor = y_true*alpha + ((1-alpha)*(1-y_true))\n        modulating_factor = K.pow((1-p_t), gamma)\n\n        # compute the final loss and return\n        return K.mean(alpha_factor*modulating_factor*bce, axis=-1)\n    return focal_crossentropy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.Sequential([\n        efn.EfficientNetB7(input_shape=(*SIZE, 3),weights='noisy-student',pooling='avg',include_top=False),\n        Dense(1, activation='sigmoid')\n    ])\n        \n    model.compile(\n        optimizer='adam',\n        loss = focal_loss(),\n        metrics=['accuracy',tf.keras.metrics.AUC(name='auc')])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Scheduler\n\n*  Modified version of the [get_cosine_schedule_with_warmup](https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup) from huggingface.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n    \"\"\"\n    Modified version of the get_cosine_schedule_with_warmup from huggingface.\n    (https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)\n\n    Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train():\n        STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\n        history = model.fit(\n            train_dataset, \n            epochs=EPOCHS, \n            callbacks=[lr_schedule],\n            steps_per_epoch=STEPS_PER_EPOCH,\n            workers=8,\n            use_multiprocessing=True,\n            validation_data=valid_dataset)\n\n        string = 'Train acc:{:.4f} Train loss:{:.4f} AUC: {:.4f}, Val acc:{:.4f} Val loss:{:.4f} Val AUC: {:.4f}'.format( \\\n            model.history.history['accuracy'][-1],model.history.history['loss'][-1],\\\n            model.history.history['auc'][-1],\\\n            model.history.history['val_accuracy'][-1],model.history.history['val_loss'][-1],\\\n            model.history.history['val_auc'][-1])\n\n        return string\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Plotting training loss, accuracy and roc","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(20,15), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_training_curves(\n    model.history.history['loss'], \n    model.history.history['val_loss'], \n    'loss', 311)\ndisplay_training_curves(\n    model.history.history['accuracy'], \n    model.history.history['val_accuracy'], \n    'accuracy', 312)\ndisplay_training_curves(\n    model.history.history['auc'], \n    model.history.history['val_auc'], \n    'auc', 313)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Prediction with TTA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames)\nsubmission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames, labeled=False,ordered=True)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(lambda image, image_name: image)\n    test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    test_pred = model.predict(test_dataset_images, verbose=1)\n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})\n    temp = submission_df.copy()\n    del temp['target']\n    submission_df['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)\npd.Series(np.round(submission_df['target'].values)).value_counts()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}