{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Melanoma Classification with Distributed TensorFlow 2.2\n\n\n**This notebook includes:**\n* Custom training loop\n* `tf.data.TFRecordDataset`/`tf.data.Dataset` with albumentations augmentation\n* Distributed training and dataset\n* Should work for CPU, Single GPU, and Multi-GPU (without having to change anything)\n* Mixed-precision (unfortunately won't have an effect here)\n* Meta features in training\n* Focal loss or BCE loss\n* Cross-validation\n* TTA\n* Learning rate schedule (warmup + decay)\n* etc.\n<br>\n\n*To modify this notebook to work with TPU, remember to exclude the current `_transform_image` function which is wrapped in tf.py_function.*\n\n**Datasets (generated by [cdeotte](https://www.kaggle.com/cdeotte), many thanks!):**\n* `melanoma-256x256`\n* `melanoma-384x384`\n* `melanoma-512x512`\n* `melanoma-768x768`\n* `melanoma-1024x1024`\n<br>\n\n**Sections:**\n\n1. [Read data](#1.-Read-data)\n2. [Dataset](#2.-Dataset)\n3. [Model](#3.-Model)\n4. [Optimizer and schedule](#4.-Optimizer-and-schedule)\n5. [Training and predicting](#5.-Training-and-predicting)\n6. [Submit test predictions](#6.-Submit-test-predictions)\n","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!pip install efficientnet -U","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom sklearn import metrics, model_selection\nimport glob\nimport os\nfrom PIL import Image\nimport tqdm.notebook as tqdm\nimport matplotlib.pyplot as plt\nimport math\nimport collections\n\nfrom efficientnet.tfkeras import (\n    EfficientNetB0, EfficientNetB1, EfficientNetB2, EfficientNetB3,\n    EfficientNetB4, EfficientNetB5, EfficientNetB6, EfficientNetB7,\n)\nfrom tensorflow.keras.applications import ResNet50, InceptionV3\n\nfrom albumentations import *\n\nmixed_precision = False\ngpus = tf.config.experimental.list_physical_devices('GPU')\nnum_gpus = len(gpus)\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(num_gpus, \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\n        \n    mixed_precision = True\n    # turn on mixed precision\n    policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    tf.keras.mixed_precision.experimental.set_policy(policy)\n    print('Compute dtype: %s' % policy.compute_dtype)\n    print('Variable dtype: %s' % policy.variable_dtype)\n\nif num_gpus == 0:\n    strategy = tf.distribute.OneDeviceStrategy(device='CPU')\n    print(\"Setting strategy to OneDeviceStrategy(device='CPU')\")\nelif num_gpus == 1:\n    strategy = tf.distribute.OneDeviceStrategy(device='GPU')\n    print(\"Setting strategy to OneDeviceStrategy(device='GPU')\")\nelse:\n    strategy = tf.distribute.MirroredStrategy()\n    print(\"Setting strategy to MirroredStrategy()\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1. Read data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"path = '../input/siim-isic-melanoma-classification/'\n\ninput_path = '../input/'\ntrain_data = pd.read_csv(path + 'train.csv')\ntest_data = pd.read_csv(path + 'test.csv')\n    \nsubmission_data = pd.read_csv(path + 'sample_submission.csv')\ntest_data['target'] = 0\nprint(\"test shape =\", test_data.shape)\nprint(test_data.head(3))\nprint(\"\\ntrain shape =\", train_data.shape)\nprint(train_data.head(3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Dataset\n\nImportant for later:\n* `create_dataset()`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"augmentor = (\n    Compose([\n        ShiftScaleRotate(\n            shift_limit=0.2,\n            scale_limit=0.2,\n            rotate_limit=180,\n            p=0.5),\n        RandomBrightnessContrast(\n            brightness_limit=0.1,\n            contrast_limit=0.1,\n            p=0.5),\n        HorizontalFlip(\n            p=0.5),\n        Transpose(\n            p=0.5),\n    ])\n)\n\n\n\ndef _parse_record(serialized, features_to_parse):\n    \n    features = {}\n    for key in features_to_parse:\n        if key == 'image_name' or key == 'image':\n            features[key] = tf.io.FixedLenFeature(\n                [], tf.string, default_value='')\n        else:\n            features[key] = tf.io.FixedLenFeature(\n                [], tf.int64, default_value=0)   \n    example = tf.io.parse_single_example(\n        serialized=serialized, features=features)\n    \n    extracted = {}\n    for key in features_to_parse:\n        if key == 'image':\n            extracted[key] = tf.io.decode_jpeg(example[key], channels=3)\n        else:\n            extracted[key] = example[key]\n    return extracted\n    \ndef _transform_image(*features):\n    '''Using albumentations augmentations, which will be \n    wrapped in tf.py_function, is very convenient. However, \n    for better performance, consider using TF operations \n    instead to augment the image data\n    '''\n    features = list(features)\n    features[0] = augmentor(image=features[0].numpy())['image']\n    return features\n\ndef _preprocess_features(features):\n    for key in features.keys():\n        if key == 'image':\n            features[key] = tf.cast(features[key], dtype=tf.float32) / 255.\n        elif key == 'anatom_site_general_challenge':\n            features[key] = tf.cast(tf.one_hot(features[key], 7), tf.float32)\n        elif key == 'diagnosis':\n            features[key] = tf.cast(tf.one_hot(features[key], 10), tf.float32)\n        elif key == 'image_name':\n            features[key] = tf.expand_dims(features[key], -1)\n        else:\n            features[key] = tf.expand_dims(tf.cast(features[key], dtype=tf.float32), -1)\n    return features\n\ndef get_dataset(tfrec_paths,\n                batch_size=16,\n                augment=False,\n                shuffle=False,\n                cache=False):\n    \n    FEATURES_TO_PARSE = [\n        'image', 'image_name', 'patient_id', \n        'target', 'anatom_site_general_challenge', \n        'sex', 'age_approx', 'diagnosis'\n    ]\n    \n    def deconstruct(features):\n        '''dict(features) --> list(features)'''\n        return list(features.values())\n    \n    def construct(*features):\n        '''list(features) --> dict(features)'''\n        return dict(zip(FEATURES_TO_PARSE, features))\n    \n    if cache:\n        if not(os.path.isdir('tmp/')):\n            os.mkdir('tmp/')\n        else:\n            files = glob.glob('tmp/*')\n            for file in files:\n                os.remove(file)\n\n        if isinstance(cache, str):\n            cache_path = 'tmp/' + cache\n        else:\n            cache_path = ''\n    \n    dataset = tf.data.TFRecordDataset(\n        filenames=tfrec_paths,\n        num_parallel_reads=tf.data.experimental.AUTOTUNE)\n\n    dataset = dataset.map(\n        lambda x: _parse_record(\n            x, features_to_parse=FEATURES_TO_PARSE),\n        tf.data.experimental.AUTOTUNE)\n\n    if cache: \n        dataset = dataset.cache(cache_path)\n\n    if shuffle: \n        dataset = dataset.shuffle(1024)\n\n    if augment:\n        dataset = dataset.map(deconstruct, tf.data.experimental.AUTOTUNE)\n        dataset = dataset.map(\n            lambda *args: tf.py_function(\n                func=_transform_image,\n                inp=args,\n                Tout=[a.dtype for a in args]),\n            tf.data.experimental.AUTOTUNE)\n        dataset = dataset.map(construct, tf.data.experimental.AUTOTUNE)\n\n    dataset = dataset.batch(batch_size)\n\n    dataset = dataset.map(_preprocess_features, tf.data.experimental.AUTOTUNE)\n\n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n\n    return dataset\n    \n    \n# Test\nds = get_dataset(\n    tfrec_paths=glob.glob(input_path+'melanoma-384x384/train[0-9]*'),\n    batch_size=16,\n    augment='heavy',\n    shuffle=False,\n    cache=None\n)\n\nfor inp in ds.take(1): pass\ndel ds\n\nfig, axes = plt.subplots(4, 4, figsize=(20, 30))\n\nfor i, ax in enumerate(axes.reshape(-1)):\n    ax.imshow(inp['image'].numpy()[i]);\n    ax.set_title(\n        ' image_name: ' + str(inp['image_name'].numpy()[i]) + '\\n' + \n        ' patient_id: ' + str(inp['patient_id'].numpy()[i]) + '\\n' + \n        ' target: '     + str(inp['target'].numpy()[i]) + '\\n' + \n        ' diagnosis: '  + str(inp['target'].numpy()[i]) + '\\n' +\n        ' site: '       + str(inp['anatom_site_general_challenge'].numpy()[i]) + '\\n' + \n        ' sex: '        + str(inp['sex'].numpy()[i]) + '\\n' + \n        ' age: '        + str(inp['age_approx'].numpy()[i]) + '\\n'\n    )\n    ax.axis('off')\n    \nplt.subplots_adjust(hspace=0.1, wspace=0.025)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Model\n\nUseful methods:\n* `dist_model.fit()`\n* `dist_model.predict()`\n* `dist_model.fit_and_predict()`\n* `dist_model.reset_weights()`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid_focal_cross_entropy_with_logits(\n    labels, logits, alpha=0.25, gamma=2.0):\n    \n    if gamma and gamma < 0:\n        raise ValueError(\"Value of gamma should be greater than or equal to zero\")\n\n    logits = tf.convert_to_tensor(logits)\n    labels = tf.convert_to_tensor(labels, dtype=logits.dtype)\n\n    # Get the cross_entropy for each entry\n    ce = tf.nn.sigmoid_cross_entropy_with_logits(\n            labels=labels, logits=logits) \n\n    # If logits are provided then convert the predictions into probabilities\n    pred_prob = tf.math.sigmoid(logits)\n    \n    p_t = (labels * pred_prob) + ((1 - labels) * (1 - pred_prob))\n    alpha_factor = 1.0\n    modulating_factor = 1.0\n\n    if alpha:\n        alpha = tf.convert_to_tensor(alpha, dtype=tf.float32)\n        alpha_factor = labels * alpha + (1 - labels) * (1 - alpha)\n\n    if gamma:\n        gamma = tf.convert_to_tensor(gamma, dtype=tf.float32)\n        modulating_factor = tf.pow((1.0 - p_t), gamma)\n\n    # compute the final loss and return\n    return tf.math.reduce_sum(\n        alpha_factor * modulating_factor * ce, axis=-1)\n\n\nclass NeuralNet(tf.keras.Model):\n    \n    def __init__(self, engine, input_shape, pretrained_weights):\n\n        super(NeuralNet, self).__init__()\n\n        self.engine = engine(\n            include_top=False,\n            input_shape=input_shape,\n            weights=pretrained_weights)\n\n        self.pool = tf.keras.layers.GlobalAveragePooling2D()\n        self.concat = tf.keras.layers.Concatenate()\n        \n        self.sequential_meta = tf.keras.Sequential([\n            tf.keras.layers.Dense(64),\n            # tf.keras.layers.BatchNormalization(),\n            tf.keras.layers.ReLU(),\n        ])\n        \n        self.sequential_merged = tf.keras.Sequential([\n            tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(1, dtype='float32')\n        ])\n        \n        \n    def call(self, inputs, **kwargs):\n        \n        if isinstance(inputs, dict):\n            images = inputs['image'] \n            site = inputs['anatom_site_general_challenge']\n            sex = inputs['sex']\n            age = inputs['age_approx']\n        else:\n            # when model.build(input_shape) is called\n            images = inputs[0]\n            site = inputs[1]\n            sex = inputs[2]\n            age = inputs[3]\n        \n        x1 = self.engine(images)\n        x1 = self.pool(x1)\n        x2 = tf.concat([site, sex, age], axis=-1)\n        x2 = self.sequential_meta(x2)\n        x3 = self.concat([x1, x2])\n        x3 = self.sequential_merged(x3)\n        return x3\n    \n    \nclass DistributedModel:\n    \n    def __init__(self, \n                 engine,\n                 input_shape=(384, 384, 3),\n                 pretrained_weights=None,\n                 finetuned_weights=None,\n                 batch_size=8,\n                 optimizer=None, \n                 strategy=None,\n                 mixed_precision=False, \n                 label_smoothing=0.0,\n                 tta=1,\n                 focal_loss=True,\n                 save_best=None):\n        \n        self.keras_model = NeuralNet(\n            engine=engine,\n            input_shape=input_shape,\n            pretrained_weights=pretrained_weights)\n        self.keras_model.build(\n            [[None, *input_shape], [None, 7], [None, 1], [None, 1]])\n        if finetuned_weights:\n            self.keras_model.load_weights(finetuned_weights)\n        self._initial_weights = self.keras_model.get_weights()\n        self.global_batch_size = batch_size\n        self.mixed_precision = mixed_precision\n        self.optimizer = optimizer\n        self.strategy = strategy\n        self.label_smoothing = label_smoothing\n        self.tta = tta\n        self.focal_loss = focal_loss\n        self.save_best = save_best\n        \n        self.auc_metric = tf.keras.metrics.AUC()\n        self.loss_metric = tf.keras.metrics.Mean()\n        \n        if self.optimizer and self.mixed_precision:\n            self.optimizer = \\\n                tf.keras.mixed_precision.experimental.LossScaleOptimizer(\n                    optimizer, loss_scale='dynamic')\n                \n        if self.strategy:\n            self.global_batch_size *= self.strategy.num_replicas_in_sync\n            \n        if save_best and not(os.path.isdir(save_best)):\n            os.makedirs(save_best) \n    \n    def reset_weights(self):\n        self.keras_model.set_weights(self._initial_weights)\n    \n    def _compute_loss(self, labels, logits):\n        if self.focal_loss:\n            per_example_loss = sigmoid_focal_cross_entropy_with_logits(\n                labels=labels, logits=logits, alpha=0.8, gamma=2.0)\n        else:\n            per_example_loss = tf.nn.sigmoid_cross_entropy_with_logits(\n                labels=labels, logits=logits) \n        return tf.nn.compute_average_loss(\n            per_example_loss, global_batch_size=self.global_batch_size)\n        \n    @tf.function\n    def _distributed_train_step(self, dist_inputs):\n        \n        def train_step(inputs):\n\n            if self.label_smoothing:\n                target = (\n                    inputs['target'] * (1 - self.label_smoothing)\n                    + 0.5 * self.label_smoothing\n                )\n            else:\n                target = inputs['target']\n\n            with tf.GradientTape() as tape:\n                \n                logits = self.keras_model(inputs, training=True)\n                loss = self._compute_loss(target, logits)\n                self.loss_metric.update_state(loss)\n                self.auc_metric.update_state(\n                    tf.math.round(target), tf.math.sigmoid(logits))\n                if self.mixed_precision:\n                    scaled_loss = self.optimizer.get_scaled_loss(loss)\n\n            if self.mixed_precision:\n                scaled_gradients = tape.gradient(\n                    scaled_loss, self.keras_model.trainable_variables)\n                gradients = self.optimizer.get_unscaled_gradients(scaled_gradients)\n            else:\n                gradients = tape.gradient(loss, self.keras_model.trainable_variables)\n\n            self.optimizer.apply_gradients(\n                zip(gradients, self.keras_model.trainable_variables))\n\n            return loss\n        \n        per_replica_loss = self.strategy.run(train_step, args=(dist_inputs,))\n        return self.strategy.reduce(\n            tf.distribute.ReduceOp.SUM, per_replica_loss, axis=None)\n        \n    @tf.function\n    def _distributed_predict_step(self, dist_inputs):\n        \n        def predict_step(inputs):\n            logits = self.keras_model(inputs, training=False)\n            return tf.math.sigmoid(logits), inputs['image_name'], inputs['target']\n    \n        preds, image_names, trues = self.strategy.run(predict_step, args=(dist_inputs,))\n        if tf.is_tensor(preds):\n            return [preds], [image_names], [trues]\n        else:\n            return preds.values, image_names.values, trues.values\n    \n    def fit(self, ds):\n\n        ds = self.strategy.experimental_distribute_dataset(ds)\n        ds = tqdm.tqdm(ds)\n        \n        for i, inputs in enumerate(ds):\n            loss = self._distributed_train_step(inputs)\n            ds.set_description(\n                \"valid AUC {:.4f} : Loss/AUC [{:.4f}, {:.4f}]\".format(\n                    self.auc_score, \n                    self.loss_metric.result().numpy(), \n                    self.auc_metric.result().numpy()\n                )\n            )\n            \n        self.loss_metric.reset_states()\n        self.auc_metric.reset_states()\n\n    def predict(self, ds):\n\n        ds = self.strategy.experimental_distribute_dataset(ds.repeat(self.tta))\n        ds = tqdm.tqdm(ds)\n        \n        preds_accum = np.zeros([0, 1], dtype=np.float32)\n        names_accum = np.zeros([0, 1], dtype=str)\n        trues_accum = np.zeros([0, 1], dtype=np.float32)\n        \n        for inputs in ds:\n            preds, names, trues = self._distributed_predict_step(inputs)\n        \n            for pred, name, true in zip(preds, names, trues):\n                preds_accum = np.concatenate([preds_accum, pred.numpy()], axis=0)\n                names_accum = np.concatenate([names_accum, name.numpy()], axis=0)\n                trues_accum = np.concatenate([trues_accum, true.numpy()], axis=0)\n        \n        preds_accum = preds_accum.reshape((self.tta, -1)).mean(axis=0)\n        names_accum = names_accum.reshape((self.tta, -1))[0]\n        trues_accum = trues_accum.reshape((self.tta, -1)).mean(axis=0).round()\n        \n        return preds_accum, names_accum, trues_accum\n    \n    def fit_and_predict(self, fold, epochs, train_ds, valid_ds, test_ds):\n        \n        self.auc_score = 0.\n        self.best_score = 0.\n        for epoch in range(epochs):\n            \n            # fit for an epoch\n            self.fit(train_ds)\n            \n            if epoch >= 9: # if statement temporary added to save time\n                # predict on validation set\n                valid_preds, valid_names, valid_trues = self.predict(valid_ds)\n\n                # compute auc score and save model if best_score\n                self.auc_score = metrics.roc_auc_score(valid_trues, valid_preds)\n\n                if self.auc_score > self.best_score:\n                    self.best_score = self.auc_score\n                    best_valid_preds = valid_preds.copy()\n                    if self.save_best:\n                        self.keras_model.save_weights(\n                            self.save_best+f'{self.keras_model.layers[0].name}-{fold}-{epoch}.h5')                \n                    # predict on test set\n                    test_preds, test_names, _ = self.predict(test_ds)\n                \n        return best_valid_preds, valid_names, test_preds, test_names ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. Optimizer and schedule\n\nImportant for later:\n* `get_optimizer()`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_optimizer(steps_per_epoch, lr_max, lr_min,\n                  decay_epochs, warmup_epochs, power=1):\n\n    if decay_epochs > 0:\n        learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(\n            initial_learning_rate=lr_max,\n            decay_steps=steps_per_epoch*decay_epochs,\n            end_learning_rate=lr_min,\n            power=power,\n        )\n    else:\n        learning_rate_fn = lr_max\n\n    if warmup_epochs > 0:\n        learning_rate_fn = WarmUp(\n            lr_start = lr_min,\n            lr_end = lr_max,\n            lr_fn = learning_rate_fn,\n            warmup_steps=steps_per_epoch*warmup_epochs,\n            power=power,\n        )\n\n    return tf.keras.optimizers.Adam(learning_rate_fn)\n\n\nclass WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule):\n\n    def __init__(self, lr_start, lr_end, lr_fn, warmup_steps, power=1):\n        super().__init__()\n        self.lr_start = lr_start\n        self.lr_end = lr_end\n        self.lr_fn = lr_fn\n        self.warmup_steps = warmup_steps\n        self.power = power\n\n    def __call__(self, step):\n        global_step_float = tf.cast(step, tf.float32)\n        warmup_steps_float = tf.cast(self.warmup_steps, tf.float32)\n        warmup_percent_done = global_step_float / warmup_steps_float\n        warmup_learning_rate = tf.add(tf.multiply(\n            self.lr_start-self.lr_end,\n            tf.math.pow(1-warmup_percent_done, self.power)), self.lr_end)\n        return tf.cond(\n            global_step_float < warmup_steps_float,\n            lambda: warmup_learning_rate,\n            lambda: self.lr_fn(step),\n        )\n\n    def get_config(self):\n        return {\n            \"lr_start\": self.lr_start,\n            \"lr_end\": self.lr_end,\n            \"lr_fn\": self.lr_fn,\n            \"warmup_steps\": self.warmup_steps,\n            \"power\": self.power,\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5. Training and predicting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"config = {\n    'lr_max': 3e-4,\n    'lr_min': 3e-5,\n    'lr_decay_epochs': 14,\n    'lr_warmup_epochs': 1,\n    'lr_decay_power': 1,\n    'n_epochs': 12,\n    'label_smoothing': 0.05,\n    'focal_loss': False,\n    'tta': 5, # if tta (> 1), shuffle has to be False\n    'save_best': None, # if not None, should be a path\n    'pretrained_weights': 'imagenet',\n    'finetuned_weights': None,\n}\n\nfold_config = {\n    0: {\n        'engine': EfficientNetB0,\n        'input_path': 'melanoma-512x512/',\n        'input_shape': (512, 512, 3),\n        'batch_size': 32,\n    },\n    1: {\n        'engine': EfficientNetB0,\n        'input_path': 'melanoma-384x384/',\n        'input_shape': (384, 384, 3),\n        'batch_size': 32,\n    },\n    2: {\n        'engine': EfficientNetB0,\n        'input_path': 'melanoma-256x256/',\n        'input_shape': (256, 256, 3),\n        'batch_size': 32,\n    },\n    3: {\n        'engine': EfficientNetB1,\n        'input_path': 'melanoma-384x384/',\n        'input_shape': (384, 384, 3),\n        'batch_size': 32,\n    },\n    4: {\n        'engine': EfficientNetB2,\n        'input_path': 'melanoma-384x384/',\n        'input_shape': (384, 384, 3),\n        'batch_size': 24,\n    },\n}\n\nsplits = model_selection.KFold(\n    len(fold_config), shuffle=True, random_state=42).split(X=range(15))\n\nwith strategy.scope():\n    \n    valid_preds_accum, test_preds_accum, test_names_accum = list(), list(), list()\n    \n    for fold, (train_idx, valid_idx) in enumerate(splits):\n        \n        optimizer = get_optimizer(\n            steps_per_epoch=33126//fold_config[fold]['batch_size'], # rough estimation\n            lr_max=config['lr_max'],\n            lr_min=config['lr_min'],\n            decay_epochs=config['lr_decay_epochs'],\n            warmup_epochs=config['lr_warmup_epochs'],\n            power=config['lr_decay_power']\n        )\n        \n        dist_model = DistributedModel(\n            engine=fold_config[fold]['engine'],\n            input_shape=fold_config[fold]['input_shape'],\n            pretrained_weights=config['pretrained_weights'],\n            finetuned_weights=config['finetuned_weights'],\n            batch_size=fold_config[fold]['batch_size'],\n            optimizer=optimizer, \n            strategy=strategy,\n            mixed_precision=mixed_precision, \n            label_smoothing=config['label_smoothing'],\n            tta=config['tta'],\n            focal_loss=config['focal_loss'],\n            save_best=config['save_best'])\n        \n        \n        tfrec_paths = np.asarray(\n            glob.glob(input_path+fold_config[fold]['input_path']+'train[0-9]*'))\n        test_paths = glob.glob(\n            input_path+fold_config[fold]['input_path']+'test[0-9]*')\n        train_paths = tfrec_paths[train_idx]\n        valid_paths = tfrec_paths[valid_idx]\n\n        train_ds = get_dataset(\n            train_paths, fold_config[fold]['batch_size'], augment=True, shuffle=True)\n        valid_ds = get_dataset(\n            valid_paths, fold_config[fold]['batch_size'], augment=True)\n        test_ds = get_dataset(\n            test_paths, fold_config[fold]['batch_size'], augment=True)\n\n        valid_preds, _, test_preds, test_names = dist_model.fit_and_predict(\n            fold=fold, \n            epochs=config['n_epochs'], \n            train_ds=train_ds, \n            valid_ds=valid_ds, \n            test_ds=test_ds, \n        )\n\n        valid_preds_accum.append(valid_preds)\n        test_preds_accum.append(test_preds)\n        test_names_accum.append(test_names)\n\n        # dist_model.reset_weights()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6. Submit test predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds = np.average(test_preds_accum, axis=0, weights=[1,1,1,1,1])\nfinal_preds_map = dict(zip(test_names_accum[0].astype('U13'), final_preds))\nsubmission_data['target'] = submission_data.image_name.map(final_preds_map)\nsubmission_data.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}