{"cells": [{"cell_type": "code", "execution_count": 1, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["%matplotlib inline\n", "%load_ext autoreload\n", "%autoreload 2"]}, {"cell_type": "code", "execution_count": 2, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["from google.cloud import storage\n", "import json\n", "import matplotlib.gridspec as gridspec\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "import pandas as pd\n", "from pathlib import Path\n", "from sklearn.metrics import classification_report\n", "import subprocess\n", "import sys\n", "import tensorflow as tf\n", "import time\n", "from tqdm.notebook import tqdm\n", "\n", "from tensorflow.keras.backend import dot"]}, {"cell_type": "code", "execution_count": 107, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["def porc(c):\n", "    print_output(run_command(c))\n", "\n", "def print_output(output):\n", "    \"\"\"Prints output from string.\"\"\"\n", "    for l in output.split('\\n'):\n", "        print(l)\n", "\n", "def print_pred_metrics(label_actual, label_pred):\n", "    \"\"\"Prints prediction evaluation metrics and report.\"\"\"\n", "    print(classification_report(label_actual, label_pred))\n", "#     print(pd.crosstab(label_actual, label_pred, margins=True))\n", "        \n", "def run_command(command):\n", "    \"\"\"Runs command line command as a subprocess returning output as string.\"\"\"\n", "    STDOUT = subprocess.PIPE\n", "    process = subprocess.run(command, shell=True, check=False,\n", "                             stdout=STDOUT, stderr=STDOUT, universal_newlines=True)\n", "    return process.stdout\n", "\n", "def show_images(imgs, titles=None, hw=(3,3), rc=(4,4)):\n", "    \"\"\"Show list of images with optional list of titles.\"\"\"\n", "    h, w = hw\n", "    r, c = rc\n", "    fig=plt.figure(figsize=(w*c, h*r))\n", "    gs1 = gridspec.GridSpec(r, c, fig, hspace=0.2, wspace=0.05)\n", "    for i in range(r*c):\n", "        img = imgs[i].squeeze()\n", "        ax = fig.add_subplot(gs1[i])\n", "        if titles != None:\n", "            ax.set_title(titles[i], {'fontsize': 10})\n", "        plt.imshow(img)\n", "        plt.axis('off')\n", "    plt.show()"]}, {"cell_type": "code", "execution_count": 4, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["output = run_command('pip freeze | grep efficientnet')\n", "if output == '':\n", "    print_output(run_command('pip install efficientnet'))\n", "else:\n", "    print_output(output)\n", "from efficientnet import tfkeras as efn"]}, {"cell_type": "code", "execution_count": 5, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["KAGGLE = os.getenv('KAGGLE_KERNEL_RUN_TYPE') != None\n", "\n", "BUCKET = 'flowers-caleb'\n", "client = storage.Client(project='fastai-caleb')\n", "bucket = client.get_bucket(BUCKET)\n", "\n", "if KAGGLE:\n", "    from kaggle_datasets import KaggleDatasets\n", "    DATASET_DIR = Path('/kaggle/input/flowers-caleb')\n", "    GCS_DATASET_DIR = KaggleDatasets().get_gcs_path(DATASET_DIR.parts[-1])\n", "    MODEL_BUCKET = GCS_DATASET_DIR.split('/')[-1]\n", "    PATH = Path('/kaggle/input/flower-classification-with-tpus')\n", "    TFRECORD_DIR = KaggleDatasets().get_gcs_path(PATH.parts[-1])\n", "    TPU_NAME = None\n", "else:\n", "    DATASET_DIR = Path('./flowers-caleb')\n", "    MODEL_BUCKET = BUCKET\n", "    PATH = Path('/home/jupyter/.fastai/data/flowers')\n", "    TFRECORD_DIR = f'gs://{BUCKET}'\n", "    TPU_NAME = 'dfdc-1'\n", "    \n", "SIZES = {s: f'{s}x{s}' for s in [192, 224, 331, 512]}\n", "\n", "AUTO = tf.data.experimental.AUTOTUNE"]}, {"cell_type": "code", "execution_count": 6, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["try:\n", "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(TPU_NAME)\n", "    print('Running on TPU ', tpu.master())\n", "except:\n", "    tpu = None\n", "\n", "if tpu:\n", "    tf.config.experimental_connect_to_cluster(tpu)\n", "    tf.tpu.experimental.initialize_tpu_system(tpu)\n", "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n", "else:\n", "    strategy = tf.distribute.get_strategy()\n", "\n", "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Datasets"]}, {"cell_type": "code", "execution_count": 7, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["# https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n", "\n", "def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n", "    # returns 3x3 transformmatrix which transforms indicies\n", "    \n", "    # CONVERT DEGREES TO RADIANS\n", "    pi = tf.constant(3.14159265359, tf.float32)\n", "    rotation = pi * rotation / 180.\n", "    shear = pi * shear / 180.\n", "    \n", "    # ROTATION MATRIX\n", "    c1 = tf.math.cos(rotation)\n", "    s1 = tf.math.sin(rotation)\n", "    one = tf.constant([1],dtype='float32')\n", "    zero = tf.constant([0],dtype='float32')\n", "    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n", "        \n", "    # SHEAR MATRIX\n", "    c2 = tf.math.cos(shear)\n", "    s2 = tf.math.sin(shear)\n", "    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n", "    \n", "    # ZOOM MATRIX\n", "    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n", "    \n", "    # SHIFT MATRIX\n", "    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n", "    \n", "    return dot(dot(rotation_matrix, shear_matrix), dot(zoom_matrix, shift_matrix))"]}, {"cell_type": "code", "execution_count": 8, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["# https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n", "\n", "def transform(image):\n", "    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n", "    # output - image randomly rotated, sheared, zoomed, and shifted\n", "    DIM = tf.shape(image)[0]\n", "    XDIM = DIM%2 #fix for size 331\n", "    \n", "    rot = 15. * tf.random.normal([1],dtype='float32')\n", "    shr = 5. * tf.random.normal([1],dtype='float32') \n", "    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n", "    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n", "    h_shift = 16. * tf.random.normal([1],dtype='float32') \n", "    w_shift = 16. * tf.random.normal([1],dtype='float32') \n", "  \n", "    # GET TRANSFORMATION MATRIX\n", "    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n", "\n", "    # LIST DESTINATION PIXEL INDICES\n", "    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n", "    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n", "    z = tf.ones([DIM*DIM],dtype='int32')\n", "    idx = tf.stack( [x,y,z] )\n", "    \n", "    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n", "    idx2 = dot(m,tf.cast(idx,dtype='float32'))\n", "    idx2 = tf.cast(idx2,dtype='int32')\n", "    idx2 = tf.clip_by_value(idx2,-DIM//2+XDIM+1,DIM//2)\n", "    \n", "    # FIND ORIGIN PIXEL VALUES           \n", "    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n", "    d = tf.gather_nd(image,tf.transpose(idx3))\n", "        \n", "    return tf.reshape(d,[DIM,DIM,3])"]}, {"cell_type": "code", "execution_count": 127, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["def augment(example):\n", "    new_example = example.copy()\n", "    image = new_example['image']\n", "#     image = transform(image)\n", "    shape = tf.shape(image)\n", "    image = tf.image.random_flip_left_right(image)\n", "#     image = tf.image.random_flip_up_down(image)\n", "    max_resize = tf.cast(tf.cast(shape[-2], tf.float32) * .2 , tf.int32)\n", "    image = tf.image.resize(image, tf.shape(image)[1:3] + tf.random.uniform((), 0, max_resize, tf.int32))\n", "    image = tf.image.random_crop(image, shape)\n", "#     image = tf.image.random_brightness(image, 0.3)\n", "#     image = tf.image.random_contrast(image, 0.9, 1.1)\n", "#     image = tf.image.random_hue(image, 0.1)\n", "#     image = tf.image.random_jpeg_quality(image, 70, 100)\n", "#     image = tf.image.random_saturation(image, 0.95, 1.05)\n", "    new_example['image'] = tf.cast(image, tf.uint8)\n", "    \n", "    return new_example\n", "\n", "def get_preprocess_fn(input_size=(224, 224), batch_size=128, norm=None, test=False):\n", "    \n", "    def imagenet_norm(image):\n", "        mean = tf.constant([0.485, 0.456, 0.406])\n", "        std = tf.constant([0.229, 0.224, 0.225])\n", "        \n", "        return (image / tf.constant(255, tf.float32) - mean) / std\n", "    \n", "    norm_fn = {'per_image': tf.image.per_image_standardization,\n", "               'imagenet': imagenet_norm,\n", "               None: tf.image.per_image_standardization\n", "              }\n", "\n", "    def preprocess(batch):\n", "        image = tf.image.resize(batch['image'], input_size)\n", "        image = norm_fn[norm](image)\n", "\n", "        if test:\n", "            return image\n", "        \n", "        else:\n", "            image = tf.reshape(image, (batch_size, *input_size, 3))\n", "            label = tf.cast(batch['label'], tf.float32)\n", "            label = tf.reshape(label, (batch_size,))\n", "                \n", "            return image, label\n", "        \n", "    return preprocess\n", "    \n", "CLASSES = tf.constant(pd.read_csv(DATASET_DIR/'classes.csv').values.squeeze(), tf.string)\n", "\n", "def get_parse_fn(split):\n", "    def parse_fn(example):\n", "        features = {\"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n", "                    \"id\": tf.io.FixedLenFeature([], tf.string),\n", "                    \"class\": tf.io.FixedLenFeature([], tf.int64)}\n", "        \n", "        if split == 'test':\n", "            del features['class']\n", "            \n", "        if split == 'ox102':\n", "            del features['id']\n", "        \n", "        example = tf.io.parse_single_example(example, features)\n", "        example['image'] = tf.image.decode_jpeg(example['image'], channels=3)\n", "        \n", "        if split != 'test':\n", "            example['label'] = tf.cast(example['class'], tf.int32)\n", "            example['class'] = CLASSES[example['label']]\n", "        return example\n", "\n", "    return parse_fn\n", "\n", "def get_ds(split, img_size=224, batch_size=128, shuffle=False):\n", "    file_pat = f'{TFRECORD_DIR}/tfrecords-jpeg-{SIZES[img_size]}/{split}/*.tfrec'\n", "    \n", "    options = tf.data.Options()\n", "    options.experimental_deterministic = not shuffle\n", "    \n", "    ds = (tf.data.Dataset.list_files(file_pat, shuffle=shuffle)\n", "          .with_options(options)\n", "          .interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n", "          .map(get_parse_fn(split), num_parallel_calls=AUTO)\n", "         )\n", "    \n", "    if split == 'train':\n", "        ds = ds.repeat().shuffle(2048).batch(batch_size).map(augment, num_parallel_calls=AUTO)\n", "        \n", "    else:\n", "        ds = ds.batch(batch_size)\n", "    \n", "    return ds.prefetch(AUTO)\n", "\n", "def get_ds_all(splits, img_size=224, batch_size=128, shuffle=True):\n", "    \"\"\"splits is a dict of {split: weight} for each split to include\"\"\"\n", "    options = tf.data.Options()\n", "    options.experimental_deterministic = not shuffle\n", "    \n", "    def get_split_ds(split):\n", "        return get_ds_for_train(split, img_size=img_size, batch_size=batch_size, shuffle=shuffle)\n", "    \n", "    ds_dict = {split: get_split_ds(split).repeat() for split in splits}\n", "\n", "    ds = tf.data.experimental.sample_from_datasets([ds_dict[s] for s in splits], [splits[s] for s in splits])\n", "        \n", "    ds = ds.shuffle(2048).batch(batch_size).map(augment, num_parallel_calls=AUTO)\n", "    \n", "    return ds.prefetch(AUTO)\n", "\n", "def get_ds_for_train(split, img_size=224, batch_size=128, shuffle=True):        \n", "    options = tf.data.Options()\n", "    options.experimental_deterministic = not shuffle\n", "    \n", "    ds = (tf.data.Dataset.list_files(get_file_pat(split, img_size), shuffle=shuffle)\n", "          .with_options(options)\n", "          .interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n", "          .map(get_parse_fn(split), num_parallel_calls=AUTO)\n", "         )\n", "    \n", "    if split == 'ox102':\n", "        ds = ds.map(add_blank_id, num_parallel_calls=AUTO)\n", "\n", "    if split == 'test':\n", "        ds = ds.filter(filter_test).map(add_test_label, num_parallel_calls=AUTO)\n", "        \n", "    return ds\n", "\n", "\n", "df_test_thresh = pd.read_csv(DATASET_DIR/'df_test_thresh.csv')\n", "TEST_IDS = tf.constant(df_test_thresh.id.values.squeeze(), tf.string)\n", "TEST_LABELS = tf.constant(df_test_thresh.label.values.squeeze(), tf.int32)\n", "\n", "def filter_test(example):\n", "    id_check = tf.math.reduce_sum(tf.cast(TEST_IDS == example['id'], tf.int32), axis=0)\n", "    return id_check > 0\n", "\n", "def add_test_label(example):\n", "    label_idx = tf.argmax(tf.cast(example['id'] == TEST_IDS, tf.int32))\n", "    example['label'] = TEST_LABELS[label_idx]\n", "    example['class'] = CLASSES[example['label']]\n", "    return example\n", "\n", "def add_blank_id(example):\n", "    example['id'] = tf.constant('unknown', tf.string)\n", "    return example\n", "\n", "def get_file_pat(split, img_size):\n", "    if split == 'ox102':\n", "        if KAGGLE:\n", "            file_pat = f'{GCS_DATASET_DIR}/extra/ox102/tfrecords-jpeg-{SIZES[img_size]}/*.tfrec'    \n", "        else:\n", "            file_pat = f'{TFRECORD_DIR}/extra/ox102/tfrecords-jpeg-{SIZES[img_size]}/*.tfrec'\n", "    else:\n", "        file_pat = f'{TFRECORD_DIR}/tfrecords-jpeg-{SIZES[img_size]}/{split}/*.tfrec'\n", "    return file_pat"]}, {"cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": ["ds = get_ds('val').unbatch().batch(16)\n", "ds_iter = iter(ds)"]}, {"cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": ["b = next(ds_iter)\n", "b_aug = augment(b)\n", "show_images(b['image'].numpy(), b['class'].numpy().tolist(), hw=(2,2), rc=(2,8))\n", "show_images(b_aug['image'].numpy(), b_aug['class'].numpy().tolist(), hw=(2,2), rc=(2,8))"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Split Distributions"]}, {"cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [], "source": ["splits = ['train', 'val', 'ox102']\n", "\n", "def split_classes(split):\n", "    return [b['label'].numpy() for b in get_ds_for_train(split)]\n", "\n", "df_split_stats = pd.concat([pd.Series(split_classes(s)).value_counts() for s in splits], axis=1).fillna(0)\n", "df_split_stats.columns = splits\n", "df_split_stats['weight'] = df_split_stats.sum(axis=1).max() / df_split_stats.sum(axis=1)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["The training and validation splits have similar distributions with respect to label. The extra ox102 samples don't have the same distribution. The class weights are calculated based on the distribution of the combined splits."]}, {"cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [], "source": ["(df_split_stats[splits].apply(lambda s: s/df_split_stats[splits].sum(axis=1))\n", ".plot(kind='area', figsize=(10,5)));"]}, {"cell_type": "markdown", "metadata": {}, "source": ["This shows the ratio of validation samples to training samples on a per class basis indexed to the overall ratio of validation samples to training samples. It looks like the distribution of samples across classes is pretty similar between the training and validation sets."]}, {"cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": ["((df_split_stats.val / df_split_stats.train) / (df_split_stats.val.sum() / df_split_stats.train.sum())).plot();"]}, {"cell_type": "markdown", "metadata": {}, "source": ["While we're at it, we'll also calculate class weights to try in training. The graph in the error analysis below shows that the f1-score is lower on the smaller classes."]}, {"cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": ["df_split_stats['weight'].plot()\n", "plt.title('Class Weights')\n", "plt.show()"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Model "]}, {"cell_type": "code", "execution_count": 143, "metadata": {}, "outputs": [], "source": ["img_size = 512 \n", "input_size = (512, 512)\n", "batch_size = 16 * strategy.num_replicas_in_sync\n", "weights = 'imagenet'\n", "\n", "ds_train = get_ds('train', img_size=img_size, batch_size=batch_size, shuffle=True)\n", "ds_valid = get_ds('val', img_size=img_size, batch_size=batch_size)\n", "\n", "splits_all = {'train': 12.7, 'ox102': 2.7}\n", "ds_train_all = get_ds_all(splits_all, img_size=img_size, batch_size=batch_size, shuffle=True)\n", "\n", "preprocess = get_preprocess_fn(batch_size=batch_size,\n", "                               input_size=input_size, norm=weights)\n", "\n", "ds_train_fit = ds_train.map(preprocess, num_parallel_calls=AUTO)\n", "ds_train_all_fit = ds_train_all.map(preprocess, num_parallel_calls=AUTO)\n", "ds_valid_fit = ds_valid.map(preprocess, num_parallel_calls=AUTO)"]}, {"cell_type": "code", "execution_count": 125, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["# Learning rate schedule for TPU, GPU and CPU.\n", "# Using an LR ramp up because fine-tuning a pre-trained model.\n", "# Starting with a high LR would break the pre-trained weights.\n", "\n", "LR_START = 0.00001\n", "LR_MAX = 0.00005 * strategy.num_replicas_in_sync \n", "LR_MIN = 0.00001\n", "LR_RAMPUP_EPOCHS = 5\n", "LR_SUSTAIN_EPOCHS = 0\n", "LR_EXP_DECAY = .7\n", "\n", "def lrfn(epoch):\n", "    if epoch < LR_RAMPUP_EPOCHS:\n", "        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n", "    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n", "        lr = LR_MAX\n", "    else:\n", "        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n", "    return lr\n", "    \n", "rng = range(20)\n", "y = [lrfn(x) for x in rng]\n", "plt.plot(rng, y)\n", "print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))"]}, {"cell_type": "code", "execution_count": 131, "metadata": {}, "outputs": [], "source": ["model_prefix = 'model_efnb7_512_08'\n", "model_dir = f'gs://{MODEL_BUCKET}/{model_prefix}'\n", "checkpoint_dir = f'{model_dir}/checkpoints'\n", "checkpoint_fn = checkpoint_dir + '/' + 'cp-{epoch:04d}.ckpt'\n", "tensorboard_cb = tf.keras.callbacks.TensorBoard(log_dir=model_dir, write_graph=False, profile_batch=0)\n", "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(checkpoint_fn, save_weights_only=True)\n", "lr_cb = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n", "latest_cp = tf.train.latest_checkpoint(checkpoint_dir)\n", "latest_cp"]}, {"cell_type": "code", "execution_count": 139, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "scrolled": true}, "outputs": [], "source": ["dataset_cp = 'cp-0011.ckpt'\n", "if False:\n", "    if not (DATASET_DIR/model_prefix).exists():\n", "        (DATASET_DIR/model_prefix).mkdir()\n", "        (DATASET_DIR/model_prefix/'checkpoints').mkdir()\n", "    \n", "    for b in bucket.list_blobs(prefix=f'{model_prefix}/checkpoints/{dataset_cp}'):\n", "        print(b.name, b.size / 1e6)\n", "        b.download_to_filename(DATASET_DIR/b.name)"]}, {"cell_type": "code", "execution_count": 130, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["if False:\n", "    print_output(run_command('rm -rf flowers-caleb/model_efnb7_512_08'))"]}, {"cell_type": "code", "execution_count": 113, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true}, "outputs": [], "source": ["model_desc = {'model_efnb7_512_06': {'model': 'efnb7, concat avg and max pool',\n", "                                             'size': 512,\n", "                                             'data': 'ds_train',\n", "                                             'aug': 'horizontal flip, vertical flip, zoom-crop 0.2',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 99,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0391,\n", "                                             'train_accuracy': 0.9897,\n", "                                             'val_loss': 0.2817,\n", "                                             'val_accuracy': 0.9440\n", "                                    },\n", "              'model_efnb7_512_07': {'model': 'efnb7, concat avg and max pool',\n", "                                             'size': 512,\n", "                                             'data': 'ds_train',\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 99,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0212,\n", "                                             'train_accuracy': 0.9949,\n", "                                             'val_loss': 0.2891,\n", "                                             'val_accuracy': 0.9448\n", "                                    },\n", "              'model_efnb7_512_08': {'model': 'efnb7, avg pool, dropout 0.2',\n", "                                             'size': 512,\n", "                                             'data': {'train': 12.7, 'val': 3.7, 'ox102': 2.7},\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 149,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 0,\n", "                                             'train_loss': 0.0073,\n", "                                             'train_accuracy': 0.9987,\n", "                                             'val_loss': 0,\n", "                                             'val_accuracy': 0\n", "                                    },\n", "              'model_efnb7_512_09': {'model': 'efnb7, avg pool',\n", "                                             'size': 512,\n", "                                             'data': 'ds_train',\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 99,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0118,\n", "                                             'train_accuracy': 0.9981,\n", "                                             'val_loss': 0.2140,\n", "                                             'val_accuracy': 0.9555\n", "                                    },\n", "              'model_efnb7_512_10': {'model': 'efnb7, avg pool, dropout 0.2',\n", "                                             'size': 512,\n", "                                             'data': 'ds_train',\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 99,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0132,\n", "                                             'train_accuracy': 0.9973,\n", "                                             'val_loss': 0.2122,\n", "                                             'val_accuracy': 0.9580\n", "                                    },\n", "              'model_efnb7_512_11': {'model': 'efnb7, avg pool, dropout 0.2',\n", "                                             'size': 512,\n", "                                             'data': {'train': 12.7, 'ox102': 2.7},\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 120,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0093,\n", "                                             'train_accuracy': 0.9985,\n", "                                             'val_loss': 0.2101,\n", "                                             'val_accuracy': 0.9599 \n", "                                    },\n", "              'model_efnb7_512_12': {'model': 'efnb7, avg pool, dropout 0.4',\n", "                                             'size': 512,\n", "                                             'data': {'train': 12.7, 'ox102': 2.7},\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 120,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0.0094,\n", "                                             'train_accuracy': 0.9986,\n", "                                             'val_loss': 0.2173,\n", "                                             'val_accuracy': 0.9582 \n", "                                    },\n", "              'model_efnb7_512_13': {'model': 'efnb7, avg pool, dropout 0.6',\n", "                                             'size': 512,\n", "                                             'data': {'train': 12.7, 'ox102': 2.7},\n", "                                             'aug': 'horizontal flip',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 100,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0,\n", "                                             'train_accuracy': 0,\n", "                                             'val_loss': 0,\n", "                                             'val_accuracy': 0 \n", "                                    }\n", "              'model_efnb7_512_14': {'model': 'efnb7, avg pool, dropout 0.2',\n", "                                             'size': 512,\n", "                                             'data': {'train': 12.7, 'ox102': 2.7},\n", "                                             'aug': 'horizontal flip, zoom crop 0.2',\n", "                                             'lr_sched': [1e-5, 5e-5 * 16, 1e-5, 5, 0, 0.8],\n", "                                             'steps': 100,\n", "                                             'epochs': 20,\n", "                                             'val_steps': 29,\n", "                                             'train_loss': 0,\n", "                                             'train_accuracy': 0,\n", "                                             'val_loss': 0,\n", "                                             'val_accuracy': 0 \n", "\n", "                     }"]}, {"cell_type": "markdown", "metadata": {}, "source": ["## Single Model"]}, {"cell_type": "code", "execution_count": 144, "metadata": {"_kg_hide-output": true, "scroll": true, "scrolled": true}, "outputs": [], "source": ["if True:\n", "    cp_to_load = f'{checkpoint_dir}/{dataset_cp}' if True else None\n", "    weights_to_load = weights if cp_to_load is None else None\n", "\n", "    with strategy.scope():\n", "\n", "        opt = tf.keras.optimizers.Adam()\n", "        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n", "\n", "        cnn = efn.EfficientNetB7(weights=weights_to_load,include_top=False,\n", "                                 pooling=None,input_shape=(*input_size, 3))\n", "        \n", "        avg_pool = tf.keras.layers.GlobalAveragePooling2D()(cnn.output)\n", "        output = avg_pool\n", "#         max_pool = tf.keras.layers.GlobalMaxPool2D()(cnn.output)\n", "#         output = tf.keras.layers.Concatenate()([avg_pool, max_pool])\n", "        output = tf.keras.layers.Dropout(0.2)(output)\n", "        output = tf.keras.layers.Dense(len(CLASSES), activation='softmax')(output)\n", "        \n", "        model = tf.keras.Model(cnn.input, output)\n", "    \n", "        if cp_to_load is not None:\n", "            model.load_weights(cp_to_load)\n", "\n", "        model.compile(loss=loss_fn, optimizer=opt, metrics=metrics)\n", "\n", "    model.summary()"]}, {"cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [], "source": ["print('split', '\\t', 'items', '\\t',  'steps')\n", "for split in ['train', 'val', 'test', 'ox102']:\n", "    items = 0\n", "    prefix = '/'.join(get_file_pat(split, img_size).split('/')[3:-1])\n", "    for b in bucket.list_blobs(prefix=prefix):\n", "        items += int(b.name.split('.')[0][-3:])\n", "#         print(b.name)\n", "    print(split, '\\t', items, '\\t',  items // batch_size)"]}, {"cell_type": "code", "execution_count": 44, "metadata": {"scrolled": false}, "outputs": [], "source": ["if False:\n", "    history = model.fit(ds_train_all_fit,\n", "                        steps_per_epoch=120,\n", "                        epochs=20,\n", "                        initial_epoch=0,\n", "                        validation_data=ds_valid_fit,\n", "                        validation_steps=29,\n", "                        callbacks=[checkpoint_cb, lr_cb],\n", "                        class_weight=df_split_stats.weight.to_list()\n", "                       )"]}, {"cell_type": "markdown", "metadata": {"heading_collapsed": true}, "source": ["## Ensemble"]}, {"cell_type": "code", "execution_count": null, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "hidden": true}, "outputs": [], "source": ["model0_prefix = 'model_efnb6_512_02'\n", "model0_dir = f'gs://{MODEL_BUCKET}/{model0_prefix}'\n", "cp0 = f'{model_dir}/checkpoints/cp-0030.ckpt'\n", "\n", "model_prefix_1 = 'model_efnb6_512_03'\n", "model_dir_1 = f'gs://{MODEL_BUCKET}/{model_prefix_1}'\n", "cp1 = f'{model_dir_1}/checkpoints/cp-0020.ckpt'"]}, {"cell_type": "code", "execution_count": null, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "hidden": true}, "outputs": [], "source": ["class EnsembleModels(tf.keras.Model):\n", "\n", "    def __init__(self, cp=None, cp1=None, load_weights=False):\n", "        super(EnsembleModels, self).__init__()\n", "        self.cnn = efn.EfficientNetB7(weights=None, include_top=False, pooling='avg')\n", "        self.cnn1 = efn.EfficientNetB7(weights=None, include_top=False, pooling='avg')\n", "        self.dense = tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n", "        self.dense1 = tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n", "        self.model = tf.keras.Sequential([self.cnn, self.dense], name='model')\n", "        self.model1 = tf.keras.Sequential([self.cnn1, self.dense1], name='model1')\n", "        self.concat = tf.keras.layers.Concatenate(name='concat')\n", "        self.post_cat_dense  = tf.keras.layers.Dense(256, activation='relu', name='post_cat_dense')\n", "        self.final = tf.keras.layers.Dense(len(CLASSES), activation='softmax', name='final')\n", "        \n", "        if load_weights:\n", "            self.model.load_weights(cp)\n", "            self.model1.load_weights(cp1)\n", "        \n", "        self.model.trainable = False\n", "        self.model1.trainable = False\n", "        \n", "    def call(self, inputs):\n", "        model_output  = self.model(inputs)\n", "        model1_output = self.model1(inputs)\n", "        output        = self.concat([model_output, model1_output])\n", "        output        = self.post_cat_dense(output)\n", "        output        = self.final(output)\n", "\n", "        return output"]}, {"cell_type": "code", "execution_count": null, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "hidden": true}, "outputs": [], "source": ["if False:\n", "    with strategy.scope():\n", "\n", "        opt = tf.keras.optimizers.Adam()\n", "        loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n", "\n", "        model = EnsembleModels(cp, cp1)\n", "\n", "        model.compile(loss=loss_fn, optimizer=opt, metrics=metrics)\n", "\n", "        model.build((batch_size, *input_size, 3))\n", "\n", "    model.summary()"]}, {"cell_type": "code", "execution_count": null, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "hidden": true, "scrolled": false}, "outputs": [], "source": ["if False:\n", "    history = model.fit(ds_train_fit,\n", "                        steps_per_epoch=99,\n", "                        epochs=10,\n", "                        initial_epoch=0,\n", "                        validation_data=ds_valid_fit,\n", "                        validation_steps=29,\n", "                        callbacks=[checkpoint_cb, tensorboard_cb, lr_cb],\n", "                        class_weight=df_split_stats.weight.to_list()\n", "                       )"]}, {"cell_type": "code", "execution_count": null, "metadata": {"_kg_hide-input": true, "_kg_hide-output": true, "hidden": true}, "outputs": [], "source": ["if False:\n", "    with strategy.scope():\n", "        model = EnsembleModels()\n", "        model.load_weights(f'{model_dir}/checkpoints/cp-0010.ckpt')"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Predictions"]}, {"cell_type": "code", "execution_count": 110, "metadata": {}, "outputs": [], "source": ["split = 'test'\n", "\n", "ds_pred = get_ds(split, img_size=img_size, batch_size=batch_size)\n", "\n", "preprocess = get_preprocess_fn(batch_size=batch_size,\n", "                               input_size=input_size, norm=weights, test=(split == 'test'))\n", "\n", "ds_pred_pp = ds_pred.map(preprocess, num_parallel_calls=AUTO)"]}, {"cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [], "source": ["# make sure example order is deterministic so we can line up training data with predictions\n", "assert np.array_equal(np.concatenate([b['id'] for b in ds_pred.as_numpy_iterator()]),\n", "               np.concatenate([b['id'] for b in ds_pred.as_numpy_iterator()]))"]}, {"cell_type": "code", "execution_count": 111, "metadata": {}, "outputs": [], "source": ["id_list = []\n", "img_list = []\n", "label_list = []\n", "class_list = []\n", "\n", "# TTA = 2\n", "# if TTA is not None:\n", "#     predictions = []\n", "#     for b in tqdm(ds_pred.take(1)):\n", "#         id_list.extend(b['id'].numpy().squeeze())\n", "#         if split == 'val':\n", "#             label_list.extend(b['label'].numpy().squeeze())\n", "#             class_list.extend(b['class'].numpy().squeeze())\n", "#         avg_preds = []\n", "#         for i in range(TTA):\n", "#             b_aug = (tf.data.Dataset.from_tensors(b).unbatch()\n", "#                      .map(augment, num_parallel_calls=AUTO).batch(batch_size)\n", "#                      .map(preprocess, num_parallel_calls=AUTO))\n", "#             preds = model.predict(b_aug)\n", "#             avg_preds.append(preds)\n", "#         predictions.extend(np.mean(np.stack(avg_preds), axis=0))\n", "#     predictions = np.stack(predictions, axis=0)\n", "# else:\n", "\n", "predictions = model.predict(ds_pred_pp)\n", "for b in ds_pred.as_numpy_iterator():\n", "    id_list.extend(b['id'].squeeze())\n", "    img_list.extend(tf.cast(tf.image.resize(b['image'], (192,192)), tf.uint8).numpy().squeeze())\n", "    if split == 'val':\n", "        label_list.extend(b['label'].squeeze())\n", "        class_list.extend(b['class'].squeeze())"]}, {"cell_type": "code", "execution_count": 112, "metadata": {}, "outputs": [], "source": ["df_pred = pd.DataFrame({'id': [n.decode() for n in id_list]})\n", "\n", "df_pred['label'] = np.argmax(predictions, axis=1)\n", "df_pred['class'] = [n.decode() for n in np.tile(np.expand_dims(CLASSES.numpy(), axis=0),\n", "                                (len(df_pred.label),))[:,df_pred.label].squeeze()]\n", "df_pred['pred_prob'] = np.take_along_axis(predictions, np.expand_dims(df_pred.label, axis=1), axis=1)\n", "    \n", "if split == 'val':\n", "    df_pred['actual_class'] = [n.decode() for n in class_list]\n", "    df_pred['actual_label'] = label_list\n", "\n", "if len(img_list) > 0:\n", "    df_pred['image'] = img_list\n", "    \n", "df_pred[['id', 'label']].to_csv('submission.csv', index=False)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Error Analysis "]}, {"cell_type": "markdown", "metadata": {"_kg_hide-input": true}, "source": ["The purpose of this analysis is to try to determine where the model can be improved. The macro F1-score is interesting in that it is an unweighted average of the class scores, meaning that the smaller classes are especially important. If your model makes errors on only a small number of images from the smaller classes, that can have a big impact on your overall score, even if you've classified many more images correctly in the bigger classes.\n", "\n", "With that in mind, the first table here is the overall F1 score. The second one, shows the class F1 scores, sorted from lowest to highest.\n", "\n", "The graph shows the class F1 scores by the number of examples in each. This model was trained using the validation data and class weights, so there isn't much, if any, bias towards the larger classes, but earlier models without the validation data and class weights indicated that the model didn't do as well on the smaller classes.\n", "\n", "The third table is the list of errors from one of the classes in the class F1 table with the images displayed below.\n", "\n", "And the last table shows the most confident correct predictions of the class along images.\n", "\n", "This version was trained on the validation data as well, so there aren't many images that were predicted incorrectly. I'm not sure there is anything to do on this one - the distinction between black-eyed susans and sunflowers is pretty fine."]}, {"cell_type": "code", "execution_count": 119, "metadata": {"_kg_hide-input": true}, "outputs": [], "source": ["error_index = 1 #change this up here to analyze errors by class, starting with classes with\n", "                #lowest f1-score\n", "\n", "if split == 'val':\n", "    class_report = classification_report(df_pred.actual_label, df_pred.label, output_dict=True)\n", "    df_cl_rep = pd.DataFrame(class_report).T\n", "    print(df_cl_rep.iloc[-3:])\n", "    df_cl_rep = df_cl_rep.iloc[:-3]\n", "    df_cl_rep = pd.DataFrame(class_report).T.iloc[:103]\n", "    df_cl_rep = df_cl_rep.sort_values('f1-score')\n", "    df_pred_g = pd.DataFrame(df_pred.groupby(['actual_label', 'label']).count()['id'])\n", "    print('\\n',df_cl_rep.head(10))\n", "\n", "    plt.scatter(df_cl_rep.support, df_cl_rep['f1-score'])\n", "    plt.title(\"Support by F1-Score\")\n", "    plt.xlabel(\"support\")\n", "    plt.ylabel(\"f1-score\")\n", "        \n", "    plt.show()\n", "    \n", "    error_label = int(df_cl_rep.index[error_index])\n", "    \n", "    df_errors = df_pred[(df_pred.actual_label == error_label) & (df_pred.label != df_pred.actual_label)].copy()\n", "    df_errors['n_class_err'] = df_errors.label.map(df_errors.groupby('label').count()['id'])\n", "    df_errors = df_errors.sort_values(['n_class_err', 'label'], ascending=[False, False])\n", "    print('\\n',df_errors[[c for c in df_errors.columns if c not in ['image', 'n_class_err']]])\n", "    \n", "    if len(img_list) > 0:\n", "        show_n = min(len(df_errors), 6)\n", "        show_images(df_errors.image.iloc[:show_n].to_list(),\n", "                    df_errors['class'].iloc[:show_n].to_list(),\n", "                    hw=(2.5,2.5), rc=(1,show_n))\n", "        \n", "        \n", "    df_correct = df_pred[(df_pred.actual_label == error_label) & (df_pred.label == df_pred.actual_label)].copy()\n", "    df_correct = df_correct.sort_values('pred_prob', ascending=False)\n", "    print('\\n',df_correct[[c for c in df_correct.columns if c not in ['image']]].head(10))\n", "\n", "    if len(img_list) > 0:\n", "        show_n = min(len(df_correct), 6)\n", "        show_images(df_correct.image.iloc[:show_n].to_list(),\n", "                    df_correct['pred_prob'].iloc[:show_n].to_list(),\n", "                    hw=(2.5,2.5), rc=(1,show_n))"]}, {"cell_type": "code", "execution_count": 122, "metadata": {}, "outputs": [], "source": ["test_index = 1 #change this up here to analyze test classes\n", "\n", "if split == 'test':\n", "    df_pred_class = df_pred.groupby('label').agg({'class': 'max', 'pred_prob': 'mean', 'id': 'count'}).sort_values('pred_prob')\n", "    print(df_pred_class.head(10))\n", "    \n", "    # change up the index here to see different classes\n", "    pred_label = df_pred_class.index[test_index]\n", "    \n", "    df_preds_for_label = df_pred[df_pred['label'] == pred_label].sort_values('pred_prob')\n", "    df_preds_for_label[['id', 'pred_prob']].set_index('id').plot(figsize=(11,5))\n", "    plt.show()\n", "    \n", "    show_n = min(6, len(df_preds_for_label))\n", "    print('low prob')\n", "    show_images(df_preds_for_label.image.to_list()[:show_n],\n", "                df_preds_for_label.pred_prob.to_list()[:show_n],\n", "                hw=(2.5,2.5),\n", "                rc=(1, show_n))\n", "    print('high prob')\n", "    show_images(df_preds_for_label.sort_values('pred_prob', ascending=False).image.to_list()[:show_n],\n", "                df_preds_for_label.sort_values('pred_prob', ascending=False).pred_prob.to_list()[:show_n],\n", "                hw=(2.5,2.5),\n", "                rc=(1, show_n))"]}, {"cell_type": "markdown", "metadata": {}, "source": ["# Push Kernel"]}, {"cell_type": "code", "execution_count": 135, "metadata": {"scrolled": true}, "outputs": [], "source": ["if False:\n", "    if not KAGGLE:\n", "        print_output(run_command(f'kaggle d version -r tar -p {DATASET_DIR} -m \"add model checkpoint\"'))"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["%%javascript\n", "IPython.notebook.save_notebook()"]}, {"cell_type": "code", "execution_count": 142, "metadata": {}, "outputs": [], "source": ["if True:\n", "    if not KAGGLE:\n", "\n", "        data = {'id': 'calebeverett/efficientnetb7-with-transformation',\n", "                      'title': 'EfficientnetB7 with Transformation',\n", "                      'code_file': 'flowers.ipynb',\n", "                      'language': 'python',\n", "                      'kernel_type': 'notebook',\n", "                      'is_private': 'false',\n", "                      'enable_gpu': 'true',\n", "                      'enable_internet': 'true',\n", "                      'dataset_sources': ['calebeverett/flowers-caleb'],\n", "                      'competition_sources': ['flower-classification-with-tpus'],\n", "                     ' kernel_sources': []}\n", "        \n", "        with open('kernel-metadata.json', 'w') as f:\n", "            json.dump(data, f)\n", "\n", "        print_output(run_command('kaggle k push'))"]}], "metadata": {"celltoolbar": "Edit Metadata", "kernelspec": {"display_name": "tf21", "language": "python", "name": "tf21"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4"}, "toc": {"base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": true, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false}}, "nbformat": 4, "nbformat_minor": 2}