{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# loading packages\n\nimport pandas as pd\nimport numpy as np\n\n#\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\n\n#\n\nimport seaborn as sns\nimport plotly.express as px\n\n#\n\nimport os\nimport random\nimport re\nimport math\nimport time\n\nfrom tqdm import tqdm\nfrom tqdm.keras import TqdmCallback\n\n\nfrom pandas_summary import DataFrameSummary\n\nimport warnings\n\n\nwarnings.filterwarnings('ignore') # Disabling warnings for clearer outputs\n\n\n\nseed_val = 42\nrandom.seed(seed_val)\nnp.random.seed(seed_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting color palette.\norange_black = [\n    '#fdc029', '#df861d', '#FF6347', '#aa3d01', '#a30e15', '#800000', '#171820'\n]\n\n# Setting plot styling.\nplt.style.use('ggplot')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting file paths for our notebook:\n\nbase_path = '/kaggle/input/siim-isic-melanoma-classification'\ntrain_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_img_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\nimg_stats_path = '/kaggle/input/melanoma2020imgtabular'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain = pd.read_csv(os.path.join(base_path, 'train.csv'))\ntest = pd.read_csv(os.path.join(base_path, 'test.csv'))\nsample = pd.read_csv(os.path.join(base_path, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\ntf.random.set_seed(seed_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading image storage buckets\n\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-384x384')\nfilenames_train = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))\nfilenames_test = np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting TPU as main device for training, if you get warnings while working with tpu's ignore them.\n\nDEVICE = 'TPU'\nif DEVICE == 'TPU':\n    print('connecting to TPU...')\n    try:        \n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print('Could not connect to TPU')\n        tpu = None\n\n    if tpu:\n        try:\n            print('Initializing  TPU...')\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print('TPU initialized')\n        except _:\n            print('Failed to initialize TPU!')\n    else:\n        DEVICE = 'GPU'\n\nif DEVICE != 'TPU':\n    print('Using default strategy for CPU and single GPU')\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == 'GPU':\n    print('Num GPUs Available: ',\n          len(tf.config.experimental.list_physical_devices('GPU')))\n\nprint('REPLICAS: ', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = dict(\n           batch_size=64,\n           img_size=384,\n    \n           lr_start=0.00001,\n           lr_max=0.00000125,\n           lr_min=0.000001,\n           lr_rampup=5,\n           lr_sustain=0,\n           lr_decay=0.8,\n           epochs=15,\n    \n           transform_prob=1.0,\n           rot=180.0,\n           shr=2.0,\n           hzoom=8.0,\n           wzoom=8.0,\n           hshift=8.0,\n           wshift=8.0,\n    \n           optimizer='adam',\n           label_smooth_fac=0.05,\n           tta_steps=20\n            \n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift,\n            width_shift):\n    \n    ''' Settings for image preparations '''\n\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n\n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1], dtype='float32')\n    zero = tf.constant([0], dtype='float32')\n    rotation_matrix = tf.reshape(\n        tf.concat([c1, s1, zero, -s1, c1, zero, zero, zero, one], axis=0),\n        [3, 3])\n\n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(\n        tf.concat([one, s2, zero, zero, c2, zero, zero, zero, one], axis=0),\n        [3, 3])\n\n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape(\n        tf.concat([\n            one / height_zoom, zero, zero, zero, one / width_zoom, zero, zero,\n            zero, one\n        ],\n                  axis=0), [3, 3])\n\n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(\n        tf.concat(\n            [one, zero, height_shift, zero, one, width_shift, zero, zero, one],\n            axis=0), [3, 3])\n\n    return K.dot(K.dot(rotation_matrix, shear_matrix),\n                 K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, cfg):\n    \n    ''' This function takes input images of [: , :, 3] sizes and returns them as randomly rotated, sheared, shifted and zoomed. '''\n\n    DIM = cfg['img_size']\n    XDIM = DIM % 2  # fix for size 331\n\n    rot = cfg['rot'] * tf.random.normal([1], dtype='float32')\n    shr = cfg['shr'] * tf.random.normal([1], dtype='float32')\n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / cfg['wzoom']\n    h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')\n    w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')\n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift)\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)\n    y = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])\n    z = tf.ones([DIM * DIM], dtype='int32')\n    idx = tf.stack([x, y, z])\n\n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)\n\n    # FIND ORIGIN PIXEL VALUES\n    idx3 = tf.stack([DIM // 2 - idx2[0, ], DIM // 2 - 1 + idx2[1, ]])\n    d = tf.gather_nd(image, tf.transpose(idx3))\n\n    return tf.reshape(d, [DIM, DIM, 3])\n\ndef prepare_image(img, cfg=None, augment=True):\n    \n    ''' This function loads the image, resizes it, casts a tensor to a new type float32 in our case, transforms it using the function just above, then applies the augmentations.'''\n    \n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [cfg['img_size'], cfg['img_size']],\n                          antialias=True)\n    img = tf.cast(img, tf.float32) / 255.0\n\n    if augment:\n        if cfg['transform_prob'] > tf.random.uniform([1], minval=0, maxval=1):\n            img = transform(img, cfg)\n\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_saturation(img, 0.7, 1.3)\n        img = tf.image.random_contrast(img, 0.8, 1.2)\n        img = tf.image.random_brightness(img, 0.1)\n\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'sex': tf.io.FixedLenFeature([], tf.int64),\n        'age_approx': tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n        'diagnosis': tf.io.FixedLenFeature([], tf.int64),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64)\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'image_name': tf.io.FixedLenFeature([], tf.string),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'sex': tf.io.FixedLenFeature([], tf.int64),\n        'age_approx': tf.io.FixedLenFeature([], tf.int64),\n        'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    return example['image'], example['image_name']\n\ndef count_data_items(filenames):\n    n = [\n        int(re.compile(r'-([0-9]*)\\.').search(filename).group(1))\n        for filename in filenames\n    ]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getTrainDataset(files, cfg, augment=True, shuffle=True):\n    \n    ''' This function reads the tfrecord train images, shuffles them, apply augmentations to them and prepares the data for training. '''\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n\n    if shuffle:\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n\n    ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.repeat()\n    if shuffle:\n        ds = ds.shuffle(2048)\n    ds = ds.map(lambda img, label:\n                (prepare_image(img, augment=augment, cfg=cfg), label),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds\n\ndef getTestDataset(files, cfg, augment=False, repeat=False):\n    \n    ''' This function reads the tfrecord test images and prepares the data for predicting. '''\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    if repeat:\n        ds = ds.repeat()\n    ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    ds = ds.map(lambda img, idnum:\n                (prepare_image(img, augment=augment, cfg=cfg), idnum),\n                num_parallel_calls=AUTO)\n    ds = ds.batch(cfg['batch_size'] * strategy.num_replicas_in_sync)\n    ds = ds.prefetch(AUTO)\n    return ds\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    \n    ''' This function gets the layers inclunding efficientnet ones. '''\n    \n    model_input = tf.keras.Input(shape=(cfg['img_size'], cfg['img_size'], 3),\n                                 name='img_input')\n\n    dummy = tf.keras.layers.Lambda(lambda x: x)(model_input)\n\n    x = efn.EfficientNetB3(include_top=False, # USE WHICHEVER YOU WANT HERE\n                           weights='noisy-student',\n                           input_shape=(cfg['img_size'], cfg['img_size'], 3),\n                           pooling='avg')(dummy) \n    x = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    model = tf.keras.Model(model_input, x)\n    model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compileNewModel(cfg, model):\n    \n    ''' Configuring the model with losses and metrics. '''    \n\n    with strategy.scope():\n        model.compile(optimizer=cfg['optimizer'],\n                      loss=[\n                          tf.keras.losses.BinaryCrossentropy(\n                              label_smoothing=cfg['label_smooth_fac'])\n                      ],\n                      metrics=[tf.keras.metrics.AUC(name='auc')])\n    return model\n\ndef getLearnRateCallback(cfg):\n    \n    ''' Using callbacks for learning rate adjustments. '''\n    \n    lr_start = cfg['lr_start']\n    lr_max = cfg['lr_max'] * strategy.num_replicas_in_sync * cfg['batch_size']\n    lr_min = cfg['lr_min']\n    lr_rampup = cfg['lr_rampup']\n    lr_sustain = cfg['lr_sustain']\n    lr_decay = cfg['lr_decay']\n\n    def lrfn(epoch):\n        if epoch < lr_rampup:\n            lr = (lr_max - lr_start) / lr_rampup * epoch + lr_start\n        elif epoch < lr_rampup + lr_sustain:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_rampup -\n                                                lr_sustain) + lr_min\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\ndef learnModel(model, ds_train, stepsTrain, cfg, ds_val=None, stepsVal=0):\n    \n    ''' Fitting things together for training '''\n    \n    callbacks = [getLearnRateCallback(cfg)]\n\n    history = model.fit(ds_train,\n                        validation_data=ds_val,\n                        verbose=1,\n                        steps_per_epoch=stepsTrain,\n                        validation_steps=stepsVal,\n                        epochs=cfg['epochs'],\n                        callbacks=callbacks)\n\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds_train = getTrainDataset(\n    filenames_train, cfg).map(lambda img, label: (img, label))#(label, label, label, label)))\nstepsTrain = count_data_items(filenames_train) / \\\n    (cfg['batch_size'] * strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = get_model()\n\nmodel = compileNewModel(cfg, model)\nhistory = learnModel(model, ds_train, stepsTrain, cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['optimizer']='sgd'\nwith strategy.scope():\n    model1 = get_model()\n\nmodel1 = compileNewModel(cfg, model1)\nhistory1 = learnModel(model1, ds_train, stepsTrain, cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2\n\n\nclass AdaBound(OptimizerV2):\n    \"\"\"AdaBound optimizer.\n    Default parameters follow those provided in the original paper.\n    # Arguments\n        learning_rate: float >= 0. Learning rate.\n        final_learning_rate: float >= 0. Final learning rate.\n        beta_1: float, 0 < beta < 1. Generally close to 1.\n        beta_2: float, 0 < beta < 1. Generally close to 1.\n        gamma: float >= 0. Convergence speed of the bound function.\n        epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n        decay: float >= 0. Learning rate decay over each update.\n        weight_decay: Weight decay weight.\n        amsbound: boolean. Whether to apply the AMSBound variant of this\n            algorithm.\n    # References\n        - [Adaptive Gradient Methods with Dynamic Bound of Learning Rate]\n          (https://openreview.net/forum?id=Bkg3g2R9FX)\n        - [Adam - A Method for Stochastic Optimization]\n          (https://arxiv.org/abs/1412.6980v8)\n        - [On the Convergence of Adam and Beyond]\n          (https://openreview.net/forum?id=ryQu7f-RZ)\n    \"\"\"\n    def __init__(self,\n                 learning_rate=0.001,\n                 final_learning_rate=0.1,\n                 beta_1=0.9,\n                 beta_2=0.999,\n                 gamma=1e-3,\n                 epsilon=None,\n                 weight_decay=0.0,\n                 amsbound=False,\n                 name='AdaBound', **kwargs):\n        super(AdaBound, self).__init__(name, **kwargs)\n\n        self._set_hyper('learning_rate', kwargs.get('learning_rate', learning_rate))\n        self._set_hyper('final_learning_rate', kwargs.get('final_learning_rate', final_learning_rate))\n        self._set_hyper('beta_1', beta_1)\n        self._set_hyper('beta_2', beta_2)\n        self._set_hyper('decay', self._initial_decay)\n        self._set_hyper('gamma', gamma)\n        self.epsilon = epsilon or tf.keras.backend.epsilon()\n        self.amsbound = amsbound\n        self.weight_decay = weight_decay\n        self.base_lr = learning_rate\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, 'm')\n            self.add_slot(var, 'v')\n            self.add_slot(var, 'vhat')\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        lr_t = self._decayed_lr(var_dtype)\n\n        m = self.get_slot(var, 'm')\n        v = self.get_slot(var, 'v')\n        vhat = self.get_slot(var, 'vhat')\n\n        beta_1_t = self._get_hyper('beta_1', var_dtype)\n        beta_2_t = self._get_hyper('beta_2', var_dtype)\n\n        gamma = self._get_hyper('gamma')\n        final_lr = self._get_hyper('final_learning_rate')\n\n        epsilon_t = tf.convert_to_tensor(self.epsilon, var_dtype)\n        base_lr_t = tf.convert_to_tensor(self.base_lr)\n        t = tf.cast(self.iterations + 1, var_dtype)\n\n        # Applies bounds on actual learning rate\n        step_size = lr_t * (tf.math.sqrt(1. - tf.math.pow(beta_2_t, t)) /\n                          (1. - tf.math.pow(beta_1_t, t)))\n\n        final_lr = final_lr * lr_t / base_lr_t\n        lower_bound = final_lr * (1. - 1. / (gamma * t + 1.))\n        upper_bound = final_lr * (1. + 1. / (gamma * t))\n\n        # apply weight decay\n        if self.weight_decay != 0.:\n            grad += self.weight_decay * var\n\n        # Compute moments\n        m_t = (beta_1_t * m) + (1. - beta_1_t) * grad\n        v_t = (beta_2_t * v) + (1. - beta_2_t) * tf.math.square(grad)\n\n        if self.amsbound:\n            vhat_t = tf.math.maximum(vhat, v_t)\n            denom = (tf.math.sqrt(vhat_t) + epsilon_t)\n        else:\n            vhat_t = vhat\n            denom = (tf.math.sqrt(v_t) + self.epsilon)\n\n        # Compute the bounds\n        step_size_p = step_size * tf.ones_like(denom)\n        step_size_p_bound = step_size_p / denom\n        bounded_lr_t = m_t * tf.math.minimum(tf.math.maximum(step_size_p_bound,\n                                             lower_bound), upper_bound)\n\n        # Setup updates\n        m_t = tf.compat.v1.assign(m, m_t)\n        vhat_t = tf.compat.v1.assign(vhat, vhat_t)\n\n        with tf.control_dependencies([m_t, v_t, vhat_t]):\n            p_t = var - bounded_lr_t\n            param_update = tf.compat.v1.assign(var, p_t)\n\n            return tf.group(*[param_update, m_t, v_t, vhat_t])\n\n    def _resource_apply_sparse(self, grad, handle, indices):\n        raise NotImplementedError(\"Sparse data is not supported yet\")\n\n    def get_config(self):\n        config = super(AdaBound, self).get_config()\n        config.update({\n            'learning_rate': self._serialize_hyperparameter('learning_rate'),\n            'final_learning_rate': self._serialize_hyperparameter('final_learning_rate'),\n            'decay': self._serialize_hyperparameter('decay'),\n            'beta_1': self._serialize_hyperparameter('beta_1'),\n            'beta_2': self._serialize_hyperparameter('beta_2'),\n            'gamma': self._serialize_hyperparameter('gamma'),\n            'epsilon': self.epsilon,\n            'weight_decay': self.weight_decay,\n            'amsbound': self.amsbound,\n        })\n        return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['optimizer'] = AdaBoundaBoundaBoost(amsbound=False)\nwith strategy.scope():\n    model2 = get_model()\n\nmodel2 = compileNewModel(cfg, model2)\nhistory2 = learnModel(model2, ds_train, stepsTrain, cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['optimizer'] = 'adam'\ncfg['epochs'] = 25\nwith strategy.scope():\n    model2 = get_model()\n\nmodel2 = compileNewModel(cfg, model2)\nhistory2 = learnModel(model2, ds_train, stepsTrain, cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['epochs'] = 1\nhistory2 = learnModel(model2, ds_train, stepsTrain, cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"auc = history2.history['auc']\nloss = history2.history['loss']\nepochs_range = range(cfg['epochs'])\n\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, auc, label='adam')\n\nplt.legend(loc='lower right')\nplt.title('Training and AUC')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='adam')\n\nplt.legend(loc='upper right')\nplt.title('Training Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"auc = history.history['auc']\nloss = history.history['loss']\nauc1 = history1.history['auc']\nloss1 = history1.history['loss']\nauc2 = history2.history['auc']\nloss2 = history2.history['loss']\nepochs_range = range(cfg['epochs'])\n\nplt.figure(figsize=(12, 6))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, auc, label='adam')\nplt.plot(epochs_range, auc1, label='sgd')\nplt.plot(epochs_range, auc2, label='adabound')\n\nplt.legend(loc='lower right')\nplt.title('Training and AUC')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='adam')\nplt.plot(epochs_range, loss1, label='sgd')\nplt.plot(epochs_range, loss2, label='adabound')\n\nplt.legend(loc='upper right')\nplt.title('Training Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg['batch_size'] = 95\nsteps = count_data_items(filenames_test) / \\\n    (cfg['batch_size'] * strategy.num_replicas_in_sync)\nz = np.zeros((cfg['batch_size'] * strategy.num_replicas_in_sync))\nds_testAug = getTestDataset(\n    filenames_test, cfg, augment=True,\n    repeat=True).map(lambda img, label: (img, z))#(z, z, z, z)))\nprobs = model2.predict(ds_testAug, verbose=1, steps=steps * cfg['tta_steps'])\nprobs = np.stack(probs)\nprobs = probs[:, :count_data_items(filenames_test) * cfg['tta_steps']]\nprobs = np.stack(np.split(probs, cfg['tta_steps']), axis=1)\nprobs = np.mean(probs, axis=1)\n\ntest = pd.read_csv(os.path.join(base_path, 'test.csv'))\ny_test_sorted = np.zeros((1, probs.shape[1]))\ntest = test.reset_index()\ntest = test.set_index('image_name')\n\n\nds_test = getTestDataset(filenames_test, cfg)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds_test.unbatch())])\nfor i in range(1):\n    submission = pd.DataFrame(dict(\n        image_name = image_names,\n        target     = probs[:,0]))\n\n    submission = submission.sort_values('image_name') \n    submission.to_csv('effnetsB.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}