{"cells":[{"metadata":{},"cell_type":"markdown","source":"Here is some code I made in order to ballance the data. Inspired from [this discussion](https://www.kaggle.com/c/flower-classification-with-tpus/discussion/130272#745018).\n\nThe idea is to use weights for each class when compiling an unballanced dataset rather than re-sampling the entire training set.\n\nPlease feel free to leave comments! Any remark, idea, suggestion, is welcome! \n\nPlease upvote if you find this notebook useful!\n\nUPDATES: \n\n1. Generating the weights take time, I generated them once and saved them, for both training set and training + validation set. Still searching for a way to do this faster and avoiding loops.\n2. The new loss function takes also a lot of time to run (model.fit), not very appropriate for this case. \n\nHere is my solution:\n"},{"metadata":{},"cell_type":"markdown","source":"<h1>Configuration and functions:</h1>\n\nFirst, re-using several lines of code from the Getting Started Notebook:"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport math, re, os\nimport numpy as np\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)\n\nfrom keras import backend as K\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.applications import *\nfrom keras import backend\nAUTO = tf.data.experimental.AUTOTUNE\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\nfrom torchvision import datasets\nfrom torchvision import transforms\n\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nif torch.cuda.is_available():\n    torch.backends.cudnn.deterministic = True\n    \n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\ntpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path() # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"IMAGE_SIZE = [512, 512] # at this size, a GPU will run out of memory. Use the TPU\nEPOCHS = 12\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = { # available image sizes\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec') # predictions on this dataset should be submitted for the competition\n\n#Various settings :     \nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\n# Took this from one of the notebooks in the competition:\nSKIP_VALIDATION = False\nif SKIP_VALIDATION:\n    TRAINING_FILENAMES = TRAINING_FILENAMES + VALIDATION_FILENAMES\n    \nSKIP_VISUALISATION = True\n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                      \n\n# Function which transforms a batch of images into numpy structures\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\n# Function which fetches the batched dataset : \ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, the function which generates a new loss function (or method), and which can be found in the discussion mentioned above, but also [here](https://gist.github.com/wassname/ce364fddfc8a025bfab4348cf5de852d)"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def weighted_categorical_crossentropy(weights):\n    \"\"\"\n    A weighted version of keras.objectives.categorical_crossentropy\n    \n    Variables:\n        weights: numpy array of shape (C,) where C is the number of classes\n    \n    Usage:\n        weights = np.array([0.5,2,10]) # Class one at 0.5, class 2 twice the normal weights, class 3 10x.\n        loss = weighted_categorical_crossentropy(weights)\n        model.compile(loss=loss,optimizer='adam')\n    \"\"\"\n    \n    weights = K.variable(weights)\n        \n    def loss(y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32) # add Version 2\n        y_pred = tf.cast(y_pred, tf.float32) # add Version 2\n        # scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n        # clip to prevent NaN's and Inf's\n        y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n        # calc\n        loss = y_true * K.log(y_pred) * weights\n        loss = -K.sum(loss, -1)\n        return loss\n    \n    return loss\n\ndef generate_nb_samples_per_class(CLASSES, NUM_TRAINING_IMAGES, train_batch):\n    # Create an empty dict in which will be stored the number of samples per class\n    n = np.zeros(len(CLASSES), dtype='float')\n    names = list(range(len(CLASSES)))\n    samples_per_classes = dict(zip(names, n))\n\n    # This is the only solution I found to stop the loop:\n    count = 0\n    while(count < NUM_TRAINING_IMAGES):\n        count = count + training_batch_size\n        im, np_labels = batch_to_numpy_images_and_labels(next(train_batch))\n    \n        # get the labels in batch, count the samples per class and add to the dict :\n        unique, counts = np.unique(np_labels, return_counts=True)\n        count_per_label = np.asarray((unique, counts)).T\n    \n        for row in count_per_label:\n            samples_per_classes[row[0]] = samples_per_classes[row[0]] + row[1]\n    \n    return samples_per_classes\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h1>Running the script</h1>"},{"metadata":{},"cell_type":"markdown","source":"Lines from the Getting Started Notebook, I just set the TRAINING_BATCH_SIZE into a constant and changed the value."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Run the function, get the training dataset : \nTRAINING_BATCH_SIZE = 200\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(TRAINING_BATCH_SIZE)\ntrain_batch = iter(training_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Apply the function to the batched dataset.\n\nI run this one for the training set and once for the training + validation set. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# samples_per_classes = generate_nb_samples_per_class(CLASSES, NUM_TRAINING_IMAGES, train_batch)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I obtain these dictionaries, for training set and training+validation set:"},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLES_PER_CLASSES_TRAINING_VALIDATION = {0: 356.0, 1: 33.0, 2: 27.0, 3: 31.0, 4: 921.0, 5: 112.0, 6: 23.0, 7: 136.0, 8: 116.0, 9: 110.0, 10: 182.0, 11: 53.0, 12: 114.0, 13: 340.0, 14: 300.0, 15: 28.0, 16: 69.0, 17: 69.0, 18: 112.0, 19: 34.0, 20: 24.0, 21: 130.0, 22: 63.0, 23: 25.0, 24: 106.0, 25: 111.0, 26: 28.0, 27: 47.0, 28: 150.0, 29: 143.0, 30: 138.0, 31: 30.0, 32: 29.0, 33: 24.0, 34: 20.0, 35: 46.0, 36: 77.0, 37: 33.0, 38: 27.0, 39: 93.0, 40: 82.0, 41: 128.0, 42: 83.0, 43: 136.0, 44: 25.0, 45: 222.0, 46: 162.0, 47: 343.0, 48: 565.0, 49: 730.0, 50: 263.0, 51: 137.0, 52: 147.0, 53: 610.0, 54: 45.0, 55: 72.0, 56: 117.0, 57: 79.0, 58: 48.0, 59: 77.0, 60: 35.0, 61: 35.0, 62: 123.0, 63: 39.0, 64: 73.0, 65: 42.0, 66: 28.0, 67: 1001.0, 68: 339.0, 69: 125.0, 70: 129.0, 71: 179.0, 72: 218.0, 73: 601.0, 74: 165.0, 75: 397.0, 76: 159.0, 77: 179.0, 78: 115.0, 79: 153.0, 80: 196.0, 81: 123.0, 82: 173.0, 83: 147.0, 84: 39.0, 85: 38.0, 86: 157.0, 87: 196.0, 88: 125.0, 89: 65.0, 90: 130.0, 91: 149.0, 92: 32.0, 93: 178.0, 94: 161.0, 95: 165.0, 96: 130.0, 97: 53.0, 98: 47.0, 99: 31.0, 100: 40.0, 101: 32.0, 102: 503.0, 103: 974.0}\n\nSAMPLES_PER_CLASSES_TRAINING = {0: 271.0, 1: 25.0, 2: 17.0, 3: 22.0, 4: 717.0, 5: 87.0, 6: 16.0, 7: 98.0, 8: 87.0, 9: 86.0, 10: 137.0, 11: 47.0, 12: 91.0, 13: 270.0, 14: 224.0, 15: 22.0, 16: 54.0, 17: 51.0, 18: 94.0, 19: 27.0, 20: 20.0, 21: 104.0, 22: 50.0, 23: 21.0, 24: 85.0, 25: 84.0, 26: 22.0, 27: 33.0, 28: 114.0, 29: 108.0, 30: 106.0, 31: 25.0, 32: 23.0, 33: 20.0, 34: 18.0, 35: 37.0, 36: 55.0, 37: 24.0, 38: 19.0, 39: 76.0, 40: 64.0, 41: 92.0, 42: 61.0, 43: 109.0, 44: 20.0, 45: 178.0, 46: 126.0, 47: 264.0, 48: 426.0, 49: 554.0, 50: 195.0, 51: 96.0, 52: 114.0, 53: 468.0, 54: 37.0, 55: 60.0, 56: 100.0, 57: 62.0, 58: 37.0, 59: 60.0, 60: 28.0, 61: 27.0, 62: 100.0, 63: 29.0, 64: 55.0, 65: 33.0, 66: 20.0, 67: 776.0, 68: 258.0, 69: 97.0, 70: 105.0, 71: 141.0, 72: 170.0, 73: 458.0, 74: 126.0, 75: 305.0, 76: 118.0, 77: 139.0, 78: 87.0, 79: 123.0, 80: 154.0, 81: 97.0, 82: 132.0, 83: 114.0, 84: 28.0, 85: 31.0, 86: 124.0, 87: 149.0, 88: 92.0, 89: 48.0, 90: 106.0, 91: 109.0, 92: 24.0, 93: 138.0, 94: 127.0, 95: 133.0, 96: 101.0, 97: 41.0, 98: 33.0, 99: 23.0, 100: 33.0, 101: 26.0, 102: 398.0, 103: 734.0}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create the Loss function:\n\n(Mind, I like using DataFrames!)\n\nI tested it : Too time consuming! "},{"metadata":{},"cell_type":"markdown","source":"\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    samples_per_classes = SAMPLES_PER_CLASSES_TRAINING\n    if SKIP_VALIDATION:\n        samples_per_classes = SAMPLES_PER_CLASSES_TRAINING_VALIDATION\n    \n    df_weights = pd.DataFrame.from_dict(samples_per_classes, orient='index', columns=['nb_samples'])\n\n    # Ballance weights for each class:  \n    sample_numbers = df_weights[df_weights['nb_samples']>0].to_numpy().flatten()\n    max_samples = np.max(sample_numbers)\n    df_weights = df_weights.applymap(lambda x: max_samples/float(x) if x>0 else 0.0)\n    weights = df_weights.to_numpy().T\n\n    # And finally :\n    home_made_loss = weighted_categorical_crossentropy(weights)\n    \n    # Let us pick this one, as an example : \n    pretrained_model = tf.keras.applications.VGG16(weights='imagenet', include_top=False ,input_shape=[*IMAGE_SIZE, 3])\n    pretrained_model.trainable = False # tramsfer learning\n    \n    # original model\n    model_original = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n    \n    model.compile(\n        optimizer='adam',\n        loss = home_made_loss,\n        metrics=['sparse_categorical_accuracy']\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(get_training_dataset(), steps_per_epoch=STEPS_PER_EPOCH, epochs=10, callbacks=[lr_callback], validation_data=get_validation_dataset())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}