{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello fellow Kagglers,\n\nThis notebook demonstrates a first attempt at segmenting cell pixels using [this](https://www.kaggle.com/markwijkhuizen/sartorius-preprocessing-kfolds-public) training data.\n\nThe training process only classifies each pixel as cell or not cell, it does not segment instances. The predicted masks could however be used as input to isolate the instance segmentation task from the segmentation task.\n\nThe model is inspired on a upsampling architecture explained in this awesome [Tensorflow Image Segmentation](https://www.tensorflow.org/tutorials/images/segmentation) tutorial.\n\n**Update V2**\n\n- All data is used for training, since the train/val metrics are approximately equal and usage of all data for training should result in a better performing model.\n- Train images do not have enhanced contrast for better performance.\n- Changed backbone model from EfficientNetV2-B1 to EfficientNetV2-S\n- The segmentation mask is used to compute the confidence levels of instances in [this](https://www.kaggle.com/markwijkhuizen/sartorius-mask-rcnn-efficientnetv2-inference) inference notebook (e.t.a. 05-12-2021)","metadata":{}},{"cell_type":"code","source":"# Silence Tensorflow\n!pip install -q silence-tensorflow\nimport silence_tensorflow.auto","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:09.681748Z","iopub.execute_input":"2021-12-04T14:56:09.682626Z","iopub.status.idle":"2021-12-04T14:56:24.988749Z","shell.execute_reply.started":"2021-12-04T14:56:09.682480Z","shell.execute_reply":"2021-12-04T14:56:24.987920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the EfficientNetV2 Library\n# Source: https://github.com/google/automl/tree/master/efficientnetv2\nimport sys\nsys.path.append('/kaggle/input/efficientnetv2-pretrained-imagenet21k-weights/brain_automl/')\nsys.path.append('/kaggle/input/efficientnetv2-pretrained-imagenet21k-weights/brain_automl/efficientnetv2/')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:24.990315Z","iopub.execute_input":"2021-12-04T14:56:24.990589Z","iopub.status.idle":"2021-12-04T14:56:24.994376Z","shell.execute_reply.started":"2021-12-04T14:56:24.990554Z","shell.execute_reply":"2021-12-04T14:56:24.993690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\nfrom sklearn import metrics\n\nimport effnetv2_model\nimport re\nimport os\nimport io\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:24.995961Z","iopub.execute_input":"2021-12-04T14:56:24.996373Z","iopub.status.idle":"2021-12-04T14:56:26.997348Z","shell.execute_reply.started":"2021-12-04T14:56:24.996334Z","shell.execute_reply":"2021-12-04T14:56:26.996608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:26.999380Z","iopub.execute_input":"2021-12-04T14:56:27.000703Z","iopub.status.idle":"2021-12-04T14:56:27.005837Z","shell.execute_reply.started":"2021-12-04T14:56:27.000658Z","shell.execute_reply":"2021-12-04T14:56:27.005179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\n# Image dimensions\nHEIGHT = 520\nWIDTH = 704\n# Image Size with padding divisable by 32 for correct upsampling\nHEIGHT_TARGET = 544\nWIDTH_TARGET = 704\nN_CHANNELS = 3\nINPUT_SHAPE = (HEIGHT_TARGET, WIDTH_TARGET, N_CHANNELS)\n\n# EfficientNet version, b0/b1/b2/b3/s/m/l/xl\nEFN_SIZE = 's'\n\n# Batch size\nBATCH_SIZE = 8\n\n# Learning Rate\nLR_MAX = 0.2\n# Number of Epochs\nEPOCHS = 20\n\n# Whether to use all data for training\nUSE_ALL_TRAINING_DATA = True\n\n# Tensorflow AUTO flag\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(f'BATCH_SIZE: {BATCH_SIZE}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:27.007136Z","iopub.execute_input":"2021-12-04T14:56:27.007678Z","iopub.status.idle":"2021-12-04T14:56:27.026805Z","shell.execute_reply.started":"2021-12-04T14:56:27.007641Z","shell.execute_reply":"2021-12-04T14:56:27.026076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Help Functions","metadata":{}},{"cell_type":"code","source":"# Random integer\n@tf.function()\ndef tf_rand_int(minval, maxval):\n    minval = tf.cast(minval, tf.int64)\n    maxval = tf.cast(maxval, tf.int64)\n    return tf.random.uniform(shape=(), minval=minval, maxval=maxval, dtype=tf.int64)\n\n# Change of 1 in K\n@tf.function()\ndef one_in(k):\n    return 0 == tf_rand_int(0, k)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:27.027923Z","iopub.execute_input":"2021-12-04T14:56:27.028265Z","iopub.status.idle":"2021-12-04T14:56:27.036313Z","shell.execute_reply.started":"2021-12-04T14:56:27.028229Z","shell.execute_reply":"2021-12-04T14:56:27.035524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# Inspiration: https://www.tensorflow.org/tutorials/generative/pix2pix#build_an_input_pipeline_with_tfdata\ndef upsample(x, concat, filters, size, name, dropout=0.0):\n    initializer = tf.random_normal_initializer(0., 0.02)\n\n    x = tf.keras.layers.Conv2DTranspose(\n            filters, # Number of Convolutional Filters\n            size, # Kernel Size\n            strides=2, # Kernel Steps\n            padding='SAME', # Keep Dimensions\n            kernel_initializer=initializer, # Weight Initializer\n            use_bias=False, # Do not use Bias only Weights\n            name=f'Conv2DTranspose_{name}' # Name of Layer\n        )(x)\n    \n    x = tf.keras.layers.BatchNormalization(name=f'BatchNormalization_{name}')(x)\n\n    if dropout > 0.0:\n        x = tf.keras.layers.Dropout(dropout, name=f'Dropout_{name}')(x)\n\n    x = tf.keras.layers.ReLU(name=f'ReLy_{name}')(x)\n    x = tf.keras.layers.Concatenate(name=f'Concatenate_{name}')([x, concat])\n\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:27.037757Z","iopub.execute_input":"2021-12-04T14:56:27.038052Z","iopub.status.idle":"2021-12-04T14:56:27.048444Z","shell.execute_reply.started":"2021-12-04T14:56:27.038018Z","shell.execute_reply":"2021-12-04T14:56:27.047510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(dropout=0.00, file_path=None):\n    tf.keras.backend.clear_session()\n    # enable XLA optmizations\n    tf.config.optimizer.set_jit(True)\n    \n    # EfficientNetV2 Backbone\n    cnn = effnetv2_model.get_model(f'efficientnetv2-{EFN_SIZE}', include_top=False, weights=None)\n    cnn.trainable = True\n\n    # Inputs, note the names are equal to the dictionary keys in the dataset\n    grayscale_image = tf.keras.layers.Input([HEIGHT_TARGET, WIDTH_TARGET, 1], name='image', dtype=tf.float32)\n\n    # CNN call, we need only the output layer\n    rgb_image = tf.keras.layers.Conv2D(3, kernel_size=1, strides=1)(grayscale_image)\n    embedding, up5, up4, up3, up2, up1 = cnn(rgb_image, with_endpoints=True)\n    print(f'embedding shape: {embedding.shape} up1 shape: {up1.shape}, up2 shape: {up2.shape}')\n    print(f'up3 shape: {up3.shape}, up4 shape: {up4.shape}, up5 shape: {up5.shape}')\n\n\n    x = upsample(up1, up2, up2.shape[-1] * 2, 3, 'upsample1_17x22', dropout=dropout)\n    x = upsample(x, up3, up3.shape[-1] * 2, 3, 'upsample2_34x44', dropout=dropout)\n    x = upsample(x, up4, up4.shape[-1] * 2, 3, 'upsample3_68x88', dropout=dropout)\n    x = upsample(x, up5, up5.shape[-1] * 2, 3, 'upsample4_136x176', dropout=dropout)\n\n    output = tf.keras.layers.Conv2DTranspose(\n            filters=1,\n            kernel_size=3,\n            strides=2,\n            padding='same',\n            activation='sigmoid'\n        )(x)\n\n\n    # We will use the famous Adam optimizer for fast learning\n    optimizer = tf.keras.optimizers.SGD(nesterov=True, momentum=0.00)\n\n    # Categorical Cross Entropy loss, from_logits=True so no softmax needed\n    loss = tf.keras.losses.BinaryCrossentropy()\n\n    # Metrics\n    metrics = [\n        tf.keras.metrics.AUC(),\n        tf.keras.metrics.BinaryAccuracy(),\n        tf.keras.metrics.Precision(),\n        tf.keras.metrics.Recall(),\n    ]\n\n    model = tf.keras.models.Model(inputs=grayscale_image, outputs=output)\n    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n    if file_path:\n        print('Loading pretrained weights...')\n        model.load_weights(file_path)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:27.050663Z","iopub.execute_input":"2021-12-04T14:56:27.050908Z","iopub.status.idle":"2021-12-04T14:56:27.067247Z","shell.execute_reply.started":"2021-12-04T14:56:27.050881Z","shell.execute_reply":"2021-12-04T14:56:27.066240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model(dropout=0.10, file_path=None)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:27.070467Z","iopub.execute_input":"2021-12-04T14:56:27.070777Z","iopub.status.idle":"2021-12-04T14:56:49.866774Z","shell.execute_reply.started":"2021-12-04T14:56:27.070741Z","shell.execute_reply":"2021-12-04T14:56:49.865958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:49.870050Z","iopub.execute_input":"2021-12-04T14:56:49.870657Z","iopub.status.idle":"2021-12-04T14:56:49.928396Z","shell.execute_reply.started":"2021-12-04T14:56:49.870625Z","shell.execute_reply":"2021-12-04T14:56:49.927644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Take a good look at the model architecture\n# The upsampling block are concatenated with the CNN filters\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:49.929591Z","iopub.execute_input":"2021-12-04T14:56:49.930228Z","iopub.status.idle":"2021-12-04T14:56:51.010387Z","shell.execute_reply.started":"2021-12-04T14:56:49.930186Z","shell.execute_reply":"2021-12-04T14:56:51.009587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"code","source":"def benchmark_dataset(dataset, num_epochs=3, n_steps_per_epoch=25, bs=BATCH_SIZE):\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch + 1)):\n            if idx == 0:\n                epoch_start = time.perf_counter()\n            elif idx == 1 and epoch_num == 0:\n                print(f'image shape: {images.shape}, image dtype: {images.dtype}')\n            else:\n                pass\n        epoch_t = time.perf_counter() - epoch_start\n        mean_step_t = round(epoch_t / n_steps_per_epoch * 1000, 1)\n        n_imgs_per_s = int(1 / (mean_step_t / 1000) * bs)\n        print(f'epoch {epoch_num} took: {round(epoch_t, 2)} sec, mean step duration: {mean_step_t}ms, images/s: {n_imgs_per_s}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.011975Z","iopub.execute_input":"2021-12-04T14:56:51.012801Z","iopub.status.idle":"2021-12-04T14:56:51.022412Z","shell.execute_reply.started":"2021-12-04T14:56:51.012758Z","shell.execute_reply":"2021-12-04T14:56:51.021772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize and pad the image and label\ndef process_image(image, label):\n    # Padding\n    pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n    pad_w = (WIDTH_TARGET - WIDTH) // 2\n    paddings = [[pad_h, pad_h], [pad_w, pad_w]]\n    \n    # Image\n    image = tf.cast(image, tf.float32)\n    image = ((image - 128) / 128) / 0.108\n    image = tf.pad(image, paddings=paddings)\n    image = tf.expand_dims(image, axis=2)\n    image = tf.repeat(image, repeats=N_CHANNELS, axis=2)\n    \n    # Label\n    label = tf.pad(label, paddings=paddings)\n    label = tf.expand_dims(label, axis=2)\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.024365Z","iopub.execute_input":"2021-12-04T14:56:51.024930Z","iopub.status.idle":"2021-12-04T14:56:51.038179Z","shell.execute_reply.started":"2021-12-04T14:56:51.024888Z","shell.execute_reply":"2021-12-04T14:56:51.037343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function(experimental_compile=True)\ndef process_image(image, label):\n    # Padding\n    pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n    pad_w = (WIDTH_TARGET - WIDTH) // 2\n    paddings = [[pad_h, pad_h], [pad_w, pad_w]]\n    \n    # Image\n    image = tf.pad(image, paddings=paddings, constant_values=128)\n    image = tf.expand_dims(image, axis=2)\n    \n    # Label\n    label = tf.pad(label, paddings=paddings, constant_values=0)\n    label = tf.expand_dims(label, axis=2)\n    \n    # Horizontal Flip\n    if one_in(2):\n        image = tf.image.flip_left_right(image)\n        label = tf.image.flip_left_right(label)\n    \n    # Vertical Flip\n    if one_in(2):\n        image = tf.image.flip_up_down(image)\n        label = tf.image.flip_up_down(label)\n        \n    # Normalise\n    image = tf.cast(image, tf.float32)\n    image = ((image - 128) / 128) / 0.589\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.040183Z","iopub.execute_input":"2021-12-04T14:56:51.040430Z","iopub.status.idle":"2021-12-04T14:56:51.053373Z","shell.execute_reply.started":"2021-12-04T14:56:51.040403Z","shell.execute_reply":"2021-12-04T14:56:51.052383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots a batch of images\ndef show_batch(dataset, rows=4, cols=4):\n    imgs, lbls = next(iter(dataset))\n    imgs = imgs.numpy()\n    # De normalise images\n    imgs = (((imgs * 0.589) * 128) + 128).astype(np.uint8)\n    # Plot\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(rows*6, cols*4))\n    for r in range(rows):\n        for c in range(cols // 2):\n            img = imgs[r*cols+c]\n            axes[r, c*2].imshow(img)\n            lbl = lbls[r*cols+c]\n            axes[r, c*2+1].imshow(lbl)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.056982Z","iopub.execute_input":"2021-12-04T14:56:51.057276Z","iopub.status.idle":"2021-12-04T14:56:51.065341Z","shell.execute_reply.started":"2021-12-04T14:56:51.057244Z","shell.execute_reply":"2021-12-04T14:56:51.064349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{}},{"cell_type":"code","source":"def get_train_dataset(fold, bs=BATCH_SIZE, print_shape=False, return_steps=False, repeat=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    X = np.load(f'/kaggle/input/sartorius-kfolds/X_fold_{fold}_train.npz')['v']\n    y = np.load(f'/kaggle/input/sartorius-kfolds/y_fold_{fold}_train.npz')['v']\n    \n    # Use all Training data\n    if USE_ALL_TRAINING_DATA:\n        X_val = np.load(f'/kaggle/input/sartorius-kfolds/X_fold_{fold}_val.npz')['v']\n        y_val = np.load(f'/kaggle/input/sartorius-kfolds/y_fold_{fold}_val.npz')['v']\n        # Concatenate\n        X = np.concatenate((X, X_val))\n        y = np.concatenate((y, y_val))\n    \n    if print_shape:\n        print(f'X shape: {X.shape}, y shape: {y.shape}')\n    \n    train_dataset = tf.data.Dataset.from_tensor_slices((X, y))\n    train_dataset = train_dataset.with_options(ignore_order)\n    train_dataset = train_dataset.shuffle(len(y), reshuffle_each_iteration=True)\n    # Don't repeat when validating training\n    if repeat:\n        train_dataset = train_dataset.repeat()\n    train_dataset = train_dataset.map(process_image, num_parallel_calls=1)\n    train_dataset = train_dataset.batch(bs)\n    train_dataset = train_dataset.prefetch(1)\n    \n    if return_steps:\n        return train_dataset, math.ceil(len(X) / bs)\n    else:\n        return train_dataset","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.068133Z","iopub.execute_input":"2021-12-04T14:56:51.068458Z","iopub.status.idle":"2021-12-04T14:56:51.081428Z","shell.execute_reply.started":"2021-12-04T14:56:51.068407Z","shell.execute_reply":"2021-12-04T14:56:51.080595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Benchmark Dataset, dataloader won't form a bottleneck\nbenchmark_dataset(get_train_dataset(0))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:51.082889Z","iopub.execute_input":"2021-12-04T14:56:51.083550Z","iopub.status.idle":"2021-12-04T14:56:58.031741Z","shell.execute_reply.started":"2021-12-04T14:56:51.083487Z","shell.execute_reply":"2021-12-04T14:56:58.030871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Image and Label Statistics as Sanity Check\nimages, labels = next(iter(get_train_dataset(0, print_shape=True)))\nprint(f'images shape: {images.shape}, labels shape: {labels.shape}')\nprint(f'images dtype: {images.dtype}, labels dtype: {labels.dtype}')\nprint(f'images min: {np.min(images):.2f}, max: {np.max(images):.2f}')\nprint(f'images mean: {np.mean(images):.2f}, std: {np.std(images):.2f}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:56:58.033207Z","iopub.execute_input":"2021-12-04T14:56:58.033938Z","iopub.status.idle":"2021-12-04T14:57:01.628255Z","shell.execute_reply.started":"2021-12-04T14:56:58.033891Z","shell.execute_reply":"2021-12-04T14:57:01.627433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot some training images\nshow_batch(get_train_dataset(0, bs=32))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:01.632775Z","iopub.execute_input":"2021-12-04T14:57:01.634900Z","iopub.status.idle":"2021-12-04T14:57:09.321255Z","shell.execute_reply.started":"2021-12-04T14:57:01.634853Z","shell.execute_reply":"2021-12-04T14:57:09.320473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Val Dataset","metadata":{}},{"cell_type":"code","source":"@tf.function(experimental_compile=True)\ndef process_image_val(image, label):\n    # Padding\n    pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n    pad_w = (WIDTH_TARGET - WIDTH) // 2\n    paddings = [[pad_h, pad_h], [pad_w, pad_w]]\n    \n    # Image\n    image = tf.pad(image, paddings=paddings, constant_values=128)\n    image = tf.expand_dims(image, axis=2)\n    image = tf.cast(image, tf.float32)\n    image = ((image - 128) / 128) / 0.589\n    \n    # Label\n    label = tf.pad(label, paddings=paddings, constant_values=0)\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.322563Z","iopub.execute_input":"2021-12-04T14:57:09.323287Z","iopub.status.idle":"2021-12-04T14:57:09.331705Z","shell.execute_reply.started":"2021-12-04T14:57:09.323249Z","shell.execute_reply":"2021-12-04T14:57:09.331042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_val_dataset(fold, bs=BATCH_SIZE, print_shape=False, return_steps=False):\n    if USE_ALL_TRAINING_DATA:\n        return (None, 0) if return_steps else None\n        \n    X = np.load(f'/kaggle/input/sartorius-kfolds/X_fold_{fold}_val.npz')['v']\n    y = np.load(f'/kaggle/input/sartorius-kfolds/y_fold_{fold}_val.npz')['v']\n    if print_shape:\n        print(f'X shape: {X.shape}, y shape: {y.shape}')\n    \n    val_dataset = tf.data.Dataset.from_tensor_slices((X, y))\n    val_dataset = val_dataset.map(process_image_val, num_parallel_calls=NUM_PARALLEL_CALLS)\n    val_dataset = val_dataset.batch(bs)\n    val_dataset = val_dataset.prefetch(1)\n    \n    if return_steps:\n        return val_dataset, len(X) // bs\n    else:\n        return val_dataset","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.333124Z","iopub.execute_input":"2021-12-04T14:57:09.334023Z","iopub.status.idle":"2021-12-04T14:57:09.347132Z","shell.execute_reply.started":"2021-12-04T14:57:09.333979Z","shell.execute_reply":"2021-12-04T14:57:09.345608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Image and Label Statistics as Sanity Check\nif not USE_ALL_TRAINING_DATA:\n    images_val, labels_val = next(iter(get_val_dataset(0, print_shape=True)))\n    print(f'images_val shape: {images_val.shape}, labels_val shape: {labels_val.shape}')\n    print(f'images_val dtype: {images_val.dtype}, labels_val dtype: {labels_val.dtype}')\n    print(f'images min: {np.min(images_val):.2f}, max: {np.max(images_val):.2f}')\n    print(f'images_val mean: {np.mean(images_val):.2f}, std: {np.std(images_val):.2f}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.348884Z","iopub.execute_input":"2021-12-04T14:57:09.349658Z","iopub.status.idle":"2021-12-04T14:57:09.357647Z","shell.execute_reply.started":"2021-12-04T14:57:09.349617Z","shell.execute_reply":"2021-12-04T14:57:09.356812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot some Validation Images\nif not USE_ALL_TRAINING_DATA:\n    show_batch(get_val_dataset(0, bs=32))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.359417Z","iopub.execute_input":"2021-12-04T14:57:09.360448Z","iopub.status.idle":"2021-12-04T14:57:09.366393Z","shell.execute_reply.started":"2021-12-04T14:57:09.360406Z","shell.execute_reply":"2021-12-04T14:57:09.365644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler\n\nLinear warmup with cosine decay, the usual. Keep in mind each epoch is 10 iterations, not 1 as conventionally. This is done to keep the logs and training history graphs readable, otherwise there would be 300 epochs.","metadata":{}},{"cell_type":"code","source":"def lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        return float(current_step + 1) / float(max(1, num_warmup_steps + 1)) * lr_max\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.369421Z","iopub.execute_input":"2021-12-04T14:57:09.369821Z","iopub.status.idle":"2021-12-04T14:57:09.376930Z","shell.execute_reply.started":"2021-12-04T14:57:09.369696Z","shell.execute_reply":"2021-12-04T14:57:09.375938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lr_schedule(lr_schedule):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(EPOCHS + 2)\n    x_axis_labels = [None] + list(map(str, np.arange(1, EPOCHS+1))) + [None]\n    plt.xlim([0, EPOCHS + 1])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if x < len(lr_schedule) - 1:\n            if lr_schedule[x - 1] < val:\n                ha = 'right'\n            else:\n                ha = 'left'\n        elif x == 0:\n            ha = 'right'\n        else:\n            ha = 'left'\n        plt.plot(x + 1, val, 'o', color='black');\n        offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n        plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=0, lr_max=LR_MAX, num_cycles=0.50) for step in range(EPOCHS)]\nplot_lr_schedule(LR_SCHEDULE)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.378413Z","iopub.execute_input":"2021-12-04T14:57:09.379092Z","iopub.status.idle":"2021-12-04T14:57:09.850350Z","shell.execute_reply.started":"2021-12-04T14:57:09.379050Z","shell.execute_reply":"2021-12-04T14:57:09.849602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks","metadata":{}},{"cell_type":"code","source":"# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.851885Z","iopub.execute_input":"2021-12-04T14:57:09.852348Z","iopub.status.idle":"2021-12-04T14:57:09.858448Z","shell.execute_reply.started":"2021-12-04T14:57:09.852299Z","shell.execute_reply":"2021-12-04T14:57:09.857430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Get the Training and Validation dataset\ntrain_dataset, train_steps_per_epoch = get_train_dataset(0, return_steps=True)\nval_dataset, val_steps_per_epoch = get_val_dataset(0, return_steps=True)\n\nprint(f'Train Steps per Epoch: {train_steps_per_epoch}')\nprint(f'Val Steps per Epoch: {val_steps_per_epoch}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:09.859970Z","iopub.execute_input":"2021-12-04T14:57:09.860354Z","iopub.status.idle":"2021-12-04T14:57:13.067001Z","shell.execute_reply.started":"2021-12-04T14:57:09.860315Z","shell.execute_reply":"2021-12-04T14:57:13.065434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model\nhistory = model.fit(\n    train_dataset,\n    # Due to low number of samples do 10 iterations per epoch\n    steps_per_epoch = train_steps_per_epoch * 10,\n    validation_data = val_dataset,\n    validation_steps = val_steps_per_epoch,\n    epochs = EPOCHS,\n    verbose = 1,\n    callbacks = [\n        lr_callback,\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T14:57:13.077279Z","iopub.execute_input":"2021-12-04T14:57:13.077772Z","iopub.status.idle":"2021-12-04T15:00:01.722727Z","shell.execute_reply.started":"2021-12-04T14:57:13.077734Z","shell.execute_reply":"2021-12-04T15:00:01.721952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model weights\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:01.724681Z","iopub.execute_input":"2021-12-04T15:00:01.725037Z","iopub.status.idle":"2021-12-04T15:00:02.346678Z","shell.execute_reply.started":"2021-12-04T15:00:01.724996Z","shell.execute_reply":"2021-12-04T15:00:02.345855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Visualisation","metadata":{}},{"cell_type":"markdown","source":"Let's check what the model learned during training. Each row consists of the input image, the actual mask, the predicted mask and the mask where each pixel is thresholded at 0.50 to form a binary map.","metadata":{}},{"cell_type":"code","source":"def plot_results(dataset, nrows, ncols=4):\n    def de_pad_batch(batch):\n        pad_h = (HEIGHT_TARGET - HEIGHT) // 2\n        pad_w = (WIDTH_TARGET - WIDTH) // 2\n        \n        return batch[:, pad_h:pad_h+HEIGHT, pad_w:pad_w+WIDTH]\n    \n    images, labels = next(iter(dataset))\n    \n    # Predict Masks\n    labels_pred = model(images, training=False)\n    \n    # Remove Padding\n    labels = de_pad_batch(labels)\n    images = (((images.numpy() * 0.108) * 128) + 128).astype(np.uint8)\n    images = de_pad_batch(images)\n    labels_pred = de_pad_batch(labels_pred)\n    \n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(ncols*8, nrows*6))\n    \n    for r, (img, lbl, lbl_pred) in enumerate(zip(images, labels, labels_pred)):\n        # Plot Image\n        axes[r, 0].imshow(img)\n        axes[r, 0].set_title('Image', size=18)\n        axes[r, 0].axis(False)\n        \n        # Mask\n        axes[r, 1].imshow(lbl)\n        axes[r, 1].set_title('Mask', size=18)\n        axes[r, 1].axis(False)\n        \n        # Predicted Mask with Threshold\n        axes[r, 2].imshow(lbl_pred)\n        axes[r, 2].set_title('Mask Predicted', size=18)\n        axes[r, 2].axis(False)\n        \n        # Predicted Mask with Threshold\n        lbl_pred_th50 =tf.cast(lbl_pred > 0.50, tf.uint8)\n        axes[r, 3].imshow(lbl_pred_th50)\n        axes[r, 3].set_title('Mask Predicted Threshold 0.50', size=18)\n        axes[r, 3].axis(False)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:02.348822Z","iopub.execute_input":"2021-12-04T15:00:02.349112Z","iopub.status.idle":"2021-12-04T15:00:02.363191Z","shell.execute_reply.started":"2021-12-04T15:00:02.349074Z","shell.execute_reply":"2021-12-04T15:00:02.362345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Visualisation, especially Astro cells have many errors\nplot_results(get_train_dataset(0, bs=16), 16)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:02.365160Z","iopub.execute_input":"2021-12-04T15:00:02.365455Z","iopub.status.idle":"2021-12-04T15:00:16.939974Z","shell.execute_reply.started":"2021-12-04T15:00:02.365415Z","shell.execute_reply":"2021-12-04T15:00:16.938623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation Visualisation\nif not USE_ALL_TRAINING_DATA:\n    plot_results(get_val_dataset(0, bs=8), 8)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:16.941480Z","iopub.execute_input":"2021-12-04T15:00:16.941983Z","iopub.status.idle":"2021-12-04T15:00:16.953404Z","shell.execute_reply.started":"2021-12-04T15:00:16.941925Z","shell.execute_reply":"2021-12-04T15:00:16.952554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax):\n    values = history.history[metric]\n    plt.figure(figsize=(15, 8))\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n    x_ticks = np.arange(1, N_EPOCHS+1)\n    \n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label='val_best')\n        plt.plot(x_ticks, val_values, label='val')\n        \n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label='train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label='train_best')\n    \n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n    plt.xlabel('epoch', fontsize=20, labelpad=10)\n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    plt.legend(prop={'size': 18})\n    plt.grid()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:16.954974Z","iopub.execute_input":"2021-12-04T15:00:16.958915Z","iopub.status.idle":"2021-12-04T15:00:16.991818Z","shell.execute_reply.started":"2021-12-04T15:00:16.958865Z","shell.execute_reply":"2021-12-04T15:00:16.990602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:16.995747Z","iopub.execute_input":"2021-12-04T15:00:16.996717Z","iopub.status.idle":"2021-12-04T15:00:17.730446Z","shell.execute_reply.started":"2021-12-04T15:00:16.996668Z","shell.execute_reply":"2021-12-04T15:00:17.729611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('binary_accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:17.735318Z","iopub.execute_input":"2021-12-04T15:00:17.737659Z","iopub.status.idle":"2021-12-04T15:00:18.251426Z","shell.execute_reply.started":"2021-12-04T15:00:17.737614Z","shell.execute_reply":"2021-12-04T15:00:18.250450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('auc')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:18.252995Z","iopub.execute_input":"2021-12-04T15:00:18.253740Z","iopub.status.idle":"2021-12-04T15:00:18.632344Z","shell.execute_reply.started":"2021-12-04T15:00:18.253697Z","shell.execute_reply":"2021-12-04T15:00:18.631428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (True Posives) / (True Positives + False Positives)\n# How many percent of predicted cell pixels are correct\n# ~80% of pixels predicted to contain a cell are correct!\nplot_history_metric('precision')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:18.635703Z","iopub.execute_input":"2021-12-04T15:00:18.636602Z","iopub.status.idle":"2021-12-04T15:00:18.997507Z","shell.execute_reply.started":"2021-12-04T15:00:18.636129Z","shell.execute_reply":"2021-12-04T15:00:18.996750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (True Positives) / (True Positives + False Negatives)\n# How many percent of the pixels containing cells are found\n# ~70% of all pixels containing a cell are found!\nplot_history_metric('recall')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:00:19.001461Z","iopub.execute_input":"2021-12-04T15:00:19.001925Z","iopub.status.idle":"2021-12-04T15:00:19.359639Z","shell.execute_reply.started":"2021-12-04T15:00:19.001878Z","shell.execute_reply":"2021-12-04T15:00:19.358897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Area Under the Receiver Operating Characteristic Curve","metadata":{}},{"cell_type":"markdown","source":"The Area Under the Receiver Operating Characeristic Curve shows the precision/recall trade off for a given threshold. This is valuable when selecting a threshold for converting prediction values in the range \\[0,1\\] to a binary mask as done with a threshold of 0.50 in the visualisations above.","metadata":{}},{"cell_type":"code","source":"def get_y_and_y_pred():\n    y = None\n    y_pred = None\n    \n    if USE_ALL_TRAINING_DATA:\n        dataset, total = get_train_dataset(0, return_steps=True, repeat=False)\n    else:\n        dataset, total = get_val_dataset(0)\n    \n    for images, label in tqdm(dataset, total=total):\n        label_pred = model(images, training=False)\n        y_batch = label.numpy().flatten().astype(np.uint8)\n        y_pred_batch = label_pred.numpy().flatten().astype(np.float32)\n        if y is None and y_pred is None:\n            y = y_batch\n            y_pred = y_pred_batch\n        else:\n            y = np.concatenate((y, y_batch), axis=0)\n            y_pred = np.concatenate((y_pred, y_pred_batch), axis=0)\n        \n    return y, y_pred\n        \ny, y_pred = get_y_and_y_pred()\n\n# Show the amount of pixels predicted, 232 million!\nprint(f'y shape: {y.shape}, y_pred shape: {y_pred.shape}')","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:01:20.966107Z","iopub.execute_input":"2021-12-04T15:01:20.966401Z","iopub.status.idle":"2021-12-04T15:01:58.485626Z","shell.execute_reply.started":"2021-12-04T15:01:20.966371Z","shell.execute_reply":"2021-12-04T15:01:58.483619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Computing Compute Receiver operating characteristic, takes several minutes...\nfalse_positive_rate, true_positive_rate, _ = metrics.roc_curve(y, y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:01:58.487574Z","iopub.execute_input":"2021-12-04T15:01:58.488016Z","iopub.status.idle":"2021-12-04T15:03:24.665676Z","shell.execute_reply.started":"2021-12-04T15:01:58.487975Z","shell.execute_reply":"2021-12-04T15:03:24.664862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.plot(false_positive_rate, true_positive_rate, color='darkorange', label='ROC')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--', label='Random Guessing')\nplt.title('Area Under the Receiver Operating Characteristic Curve', size=24)\nplt.ylabel('True Positive Rate', size=18, labelpad=10)\nplt.xlabel('False Positive Rate', size=18, labelpad=10)\nplt.xticks(size=16)\nplt.yticks(size=16)\nplt.grid()\nplt.legend(prop={'size': 16})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:03:24.667022Z","iopub.execute_input":"2021-12-04T15:03:24.667289Z","iopub.status.idle":"2021-12-04T15:03:44.409161Z","shell.execute_reply.started":"2021-12-04T15:03:24.667254Z","shell.execute_reply":"2021-12-04T15:03:44.408425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove false_positive_rate and true_positive_rate to reduce memory usage\ndel false_positive_rate, true_positive_rate\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:04:08.952309Z","iopub.execute_input":"2021-12-04T15:04:08.953167Z","iopub.status.idle":"2021-12-04T15:04:09.282906Z","shell.execute_reply.started":"2021-12-04T15:04:08.953107Z","shell.execute_reply":"2021-12-04T15:04:09.281799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Precision/Recall/Threshold Curve","metadata":{}},{"cell_type":"code","source":"# Compute Precision/Recall Curve, takes several minutes...\nprecision, recall, thresholds = metrics.precision_recall_curve(y, y_pred)\nthresholds = np.concatenate(([0], thresholds))","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:04:23.943242Z","iopub.execute_input":"2021-12-04T15:04:23.943499Z","iopub.status.idle":"2021-12-04T15:05:48.841512Z","shell.execute_reply.started":"2021-12-04T15:04:23.943470Z","shell.execute_reply":"2021-12-04T15:05:48.840679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Precision/Recall curve shows the precision (fraction of pixels predicted as cell pixels that actually are cell pixels) at a given recall (fraction of cell pixels predicted as cell pixels).","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.plot(precision, recall, color='darkorange', label='Precision/Recall')\nplt.title('Precision/Recall Curve', size=24)\nplt.xlabel('Precision', size=18, labelpad=10)\nplt.ylabel('Recall', size=18, labelpad=10)\nplt.xticks(size=16)\nplt.yticks(size=16)\nplt.grid()\nplt.legend(prop={'size': 16})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:05:48.843259Z","iopub.execute_input":"2021-12-04T15:05:48.843595Z","iopub.status.idle":"2021-12-04T15:06:02.017970Z","shell.execute_reply.started":"2021-12-04T15:05:48.843536Z","shell.execute_reply":"2021-12-04T15:06:02.016585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Threshold/Recall Curve shows the recall value at a given threshold, this allows to pick a threshold for a desired recall value.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.plot(recall, thresholds,  color='darkorange', label='Recall/Threshold')\nplt.title('Threshold/Recall Curve', size=24)\nplt.xlabel('Threshold', size=18, labelpad=10)\nplt.ylabel('Recall', size=18, labelpad=10)\nplt.xticks(size=16)\nplt.yticks(size=16)\nplt.grid()\nplt.legend(prop={'size': 16})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:06:02.019318Z","iopub.execute_input":"2021-12-04T15:06:02.020161Z","iopub.status.idle":"2021-12-04T15:06:15.179389Z","shell.execute_reply.started":"2021-12-04T15:06:02.020117Z","shell.execute_reply":"2021-12-04T15:06:15.178485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Threshold/Precision Curve shows the precision value at a given threshold, this allows to pick a threshold for a desired precision value.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.plot(thresholds, precision,  color='darkorange', label='Precision/Threshold')\nplt.title('Threshold/Precision Curve', size=24)\nplt.xlabel('Threshold', size=18, labelpad=10)\nplt.ylabel('Precision', size=18, labelpad=10)\nplt.xticks(size=16)\nplt.yticks(size=16)\nplt.grid()\nplt.legend(prop={'size': 16})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-04T15:06:15.181302Z","iopub.execute_input":"2021-12-04T15:06:15.182178Z","iopub.status.idle":"2021-12-04T15:06:37.155113Z","shell.execute_reply.started":"2021-12-04T15:06:15.182135Z","shell.execute_reply":"2021-12-04T15:06:37.154360Z"},"trusted":true},"execution_count":null,"outputs":[]}]}