{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport re\nimport cv2\nimport math\nimport time\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom plotly.subplots import make_subplots\nimport plotly.express as px","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configuration\n\nreference: https://www.kaggle.com/hatemamine/flowertpuwin","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Tensorflow version \" + tf.__version__)\n\nAUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n     \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nDATASET = '512x512-melanoma-tfrecords-70k-images'\nGCS_PATH = KaggleDatasets().get_gcs_path(DATASET)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Cleaning & EDA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data  = train_data.dropna()\n\ntrain_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualising Train DataSet \nfig = px.histogram(train_data, x=\"benign_malignant\")\nfig.update_layout(title_text='Benign and Malignant cases in dataset')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# anatom_site_general_challenge\nfig = px.histogram(train_data, x=\"anatom_site_general_challenge\")\nfig.update_layout(title_text='Anatom Sites')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sex \nfig = px.histogram(train_data, x=\"sex\")\nfig.update_layout(title_text='Affected Male / Female counts')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Parallel Categories diagram on train dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"parallel_diagram = train_data[['image_name', 'patient_id', 'sex', 'anatom_site_general_challenge', 'diagnosis', 'target']]\n\nfig = px.parallel_categories(parallel_diagram, color=\"target\",  color_continuous_scale=px.colors.sequential.Inferno)\nfig.update_layout(title='Parallel category diagram on trainset')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparameter Tuning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nSIZE = [512,512]\nLR = 0.00004\nEPOCHS = 12\nWARMUP = 5\nWEIGHT_DECAY = 0\nLABEL_SMOOTHING = 0.05\nTTA = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.2,random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32)/255.0\n    image = tf.reshape(image, [*SIZE, 3])\n    return image\n\ndef data_augment(image, label=None, seed=SEED):\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string), }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    return image, image_name\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n\n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n              .with_options(ignore_order)\n              .map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO))\n            \n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\ndef plot_transform(num_images):\n    plt.figure(figsize=(30,10))\n    x = load_dataset(train_filenames, labeled=False)\n    image,_ = iter(x).next()\n    for i in range(1,num_images+1):\n        plt.subplot(1,num_images+1,i)\n        plt.axis('off')\n        image = data_augment(image=image)\n        plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_transform(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (load_dataset(train_filenames, labeled=True)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (load_dataset(valid_filenames, labeled=True)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNetB7 \n\nrefer: https://keras.io/api/applications/efficientnet/","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n\n    model = tf.keras.Sequential([\n        efn.EfficientNetB7(input_shape=(*SIZE, 3),weights='imagenet',pooling='avg',include_top=False),\n        Dense(1, activation='sigmoid')\n    ])\n    \n    model.compile(\n        optimizer='adam',\n        loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),\n        metrics=['accuracy',tf.keras.metrics.AUC(name='auc')])\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate Schedules\nreference: https://huggingface.co/transformers/main_classes/optimizer_schedules.html","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return (float(epoch) / float(max(1, num_warmup_steps))) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train():\n        STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\n        history = model.fit(\n            train_dataset, \n            epochs=EPOCHS, \n            callbacks=[lr_schedule],\n            steps_per_epoch=STEPS_PER_EPOCH,\n            validation_data=valid_dataset)\n\n        string = 'Train acc:{:.4f} Train loss:{:.4f} AUC: {:.4f}, Val acc:{:.4f} Val loss:{:.4f} Val AUC: {:.4f}'.format( \\\n            model.history.history['accuracy'][-1],model.history.history['loss'][-1],\\\n            model.history.history['auc'][-1],\\\n            model.history.history['val_accuracy'][-1],model.history.history['val_loss'][-1],\\\n            model.history.history['val_auc'][-1])\n\n        return string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction Based on Test Time Augmentation (TTA)\n- Data augmentation technique for the test dataset, where multiple augmentaed copies of images in dataset is created with zoom, flip and shifts\n- The artificially expanded training dataset can result in a more skillful model, as often the performance of deep learning models continues to scale in concert with the size of the training dataset\n- The model makes prediction for each and then ensemble of the predictions are returned\n\nsource - [TTA - Machinelearningmastery](https://machinelearningmastery.com/how-to-use-test-time-augmentation-to-improve-model-performance-for-image-classification/#:~:text=Test%2Dtime%20augmentation%2C%20or%20TTA,an%20ensemble%20of%20those%20predictions.) ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames)\nsubmission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nfor i in range(TTA):\n    test_dataset = (load_dataset(test_filenames, labeled=False,ordered=True)\n    .map(data_augment, num_parallel_calls=AUTO)  \n    .batch(BATCH_SIZE))\n    test_dataset_images = test_dataset.map(lambda image, image_name: image)\n    test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\n    test_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')\n    test_pred = model.predict(test_dataset_images, verbose=1) \n    pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})\n    temp = submission_df.copy()   \n    del temp['target']  \n    submission_df['target'] += temp.merge(pred_df,on=\"image_name\")['target']/TTA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)\npd.Series(np.round(submission_df['target'].values)).value_counts()  \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}