{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# These are some basic packages\nimport random, re, math, os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport random\n\n# data processing\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets\n\n\n# model training\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom tensorflow.keras.applications import DenseNet201\nimport tensorflow.keras.backend as K\n\n\n# These are performance metrics\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\n\n# These are for class weights\nimport datetime\nimport tqdm\nimport json\nfrom collections import Counter\nimport gc\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Previous versions\n\n* In the first two versions of this notebook, I tried to achieve at least some results, armed with the ResNet architecture.\n\n* In the third version, I got about 0.9459 using DenseNet and raw data.\n\n* For the fourth time, I tried to make an ensemble from DenseNet and ResNet with training on augmented images, and for this I received 0.4988. Shame!\n\nIn this version I trying to increase my own score obtained with DenseNet201 and raw data.","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\n#seed = 0\n#seed_everything(seed)\n\n#I decided not to use seed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use power of mixed precision for better capacity\n\nMIXED_PRECISION = False\nXLA_ACCELERATE = False\n\nif MIXED_PRECISION:\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#classes names for vizualization\n\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose'] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE #autotune for tf.data.Dataset.prefetch\n\nIMAGE_SIZE = [512, 512]\nEPOCHS = 16\nSEED = 100\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nCHANNELS = 3\n\nHEIGHT = 512\nWIDTH = 512","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\n# These are available image sizes in the data set\nGCS_PATH_SELECT = { \n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions for preparing datasets, augmentation and visualization of our flowers","metadata":{}},{"cell_type":"code","source":"#visualization function for our metrics\n\ndef plot_train_valid_curves(training, validation, title, subplot):\n    \n    if subplot % 10 == 1:\n        plt.subplots(figsize = (15,15), facecolor = '#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['training', 'validation.'])","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    # Set seed for data augmentation\n    seed = 100\n    \n    # Set seed for data augmentation\n    seed = 100\n    \n    # Randomly resize and then crop images\n    image = tf.image.resize(image, [720, 720])\n    image = tf.image.random_crop(image, [512, 512, 3], seed = seed)\n\n    # Randomly reset brightness of images\n    #image = tf.image.random_brightness(image, 0.3, seed = seed)\n    \n    # Randomly reset saturation of images\n    image = tf.image.random_saturation(image, 0, 1, seed = seed)\n        \n    # Randomly reset contrast of images\n    image = tf.image.random_contrast(image, 0.6, 1, seed = seed)\n    \n    # Blur images \n    #image = tfa.image.mean_filter2d(image, filter_shape = 10)\n    \n    # Randomly flip images\n    image = tf.image.random_flip_left_right(image, seed = seed)\n    image = tf.image.random_flip_up_down(image, seed = seed)\n    \n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(lambda image, label: [image, label])\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset_preview(ordered=True):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n# Visualize model predictions\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', shoud be ' if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\ntrain_dataset = get_training_dataset_preview(ordered=True)\ny_train = next(iter(train_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\nprint(f'Number of training images {NUM_TRAINING_IMAGES}')\n\n# Validation data\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nvalid_dataset = get_validation_dataset(ordered=True)\ny_valid = next(iter(valid_dataset.unbatch().map(lambda image, label: label).batch(NUM_VALIDATION_IMAGES))).numpy()\nprint(f'Number of validation images {NUM_VALIDATION_IMAGES}')\n\n# Test data\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(f'Number of test images {NUM_TEST_IMAGES}')\ntest_dataset = get_test_dataset(ordered=True)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Admiring flowers","metadata":{}},{"cell_type":"code","source":"row = 6\ncol = 6\nall_elements = get_training_dataset_preview().unbatch()\none_element = tf.data.Dataset.from_tensors(next(iter(all_elements)))\n# Map the images to the data augmentation function for image processing\naugmented_element = one_element.repeat().map(data_augment).batch(row * col)\n\nfor (img, label) in augmented_element:\n    plt.figure(figsize = (15, int(15 * row / col)))\n    for j in range(row * col):\n        plt.subplot(row, col, j + 1)\n        plt.axis('off')\n        plt.imshow(img[j, ])\n    plt.show()\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_aug = get_training_dataset()\ndisplay_batch_of_images(next(iter(train_dataset_aug.unbatch().batch(20))))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_dataset_aug = get_validation_dataset()\ndisplay_batch_of_images(next(iter(validation_dataset_aug.unbatch().batch(20))))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset_aug = get_test_dataset()\ndisplay_batch_of_images(next(iter(test_dataset_aug.unbatch().batch(20))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"def lrfn(epoch):\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY ** (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\n# Visualization changes in learning rate\nrng = [i for i in range(20 if EPOCHS<20 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mystery\n\nTransfer learning is not working here...","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    # Create DenseNet201 model\n    densenet = tf.keras.applications.DenseNet201(\n        input_shape = (512, 512, 3),\n        weights = 'imagenet',  # Use the preset parameters of ImageNet\n        include_top = False  # Drop the fully connected network on the top\n    )\n    \n    densenet.trainable =True\n    model_dens = tf.keras.Sequential([\n        densenet,\n        #tf.keras.layers.Dropout(0.2),\n        #tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation = 'softmax')\n    ])\n\n    model_dens.compile(\n        optimizer=tf.keras.optimizers.Adam(), loss = 'sparse_categorical_crossentropy', \n        metrics = ['sparse_categorical_accuracy']\n    )\n    model_dens.summary()\n    model_dens.save('ML_1.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As my experience has shown, if we do not calculate the weights for the classes separately, the model will show a much worse result","metadata":{}},{"cell_type":"code","source":"gc.enable()\n\ndef get_training_dataset_raw():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled = True, ordered = False)\n    return dataset\n\nraw_training_dataset = get_training_dataset_raw()\n\nlabel_counter = Counter()\nfor images, labels in raw_training_dataset:\n    label_counter.update([labels.numpy()])\n\ndel raw_training_dataset    \n\nTARGET_NUM_PER_CLASS = 122\n\ndef get_weight_for_class(class_id):\n    counting = label_counter[class_id]\n    weight = TARGET_NUM_PER_CLASS / counting\n    return weight\n\nweight_per_class = {class_id: get_weight_for_class(class_id) for class_id in range(104)}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 16\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_dens.fit(\n    get_training_dataset(),\n    steps_per_epoch = STEPS_PER_EPOCH,\n    epochs = EPOCHS,\n    callbacks = [lr_callback],\n    validation_data = get_validation_dataset(),\n    validation_steps=VALID_STEPS,\n    class_weight = weight_per_class\n)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_train_valid_curves(history.history['loss'], history.history['val_loss'], 'loss', 211) \nplot_train_valid_curves(history.history['sparse_categorical_accuracy'], \n                        history.history['val_sparse_categorical_accuracy'], 'accuracy', 212) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# My experience with image augmentation\n\nIt turned out that no matter how high-quality augmentation was, the validation metrics still fall lower than in raw images.\n\nFor example, with augmentation, which was visualized above, the accuracy on the validation data was only 0.92, and in other cases it sometimes did not rise even to 0.91. It's very hard for me to explain it somehow.\n\nThe author of [this notebook](https://www.kaggle.com/xuanzhihuang/flower-classification-densenet-201/comments) came to exactly the same conclusion.","metadata":{}},{"cell_type":"markdown","source":"# Model selection\nOf course, when choosing the second model for the ensemble, I was guided by [this wonderful work](https://www.kaggle.com/georgezoto/computer-vision-petals-to-the-metal).","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet\n\nimport efficientnet.tfkeras as efn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    enet = efn.EfficientNetB7(\n        input_shape=(512, 512, 3),\n        weights='noisy-student',\n        include_top=False\n    )\n\n    model_enet = tf.keras.Sequential([\n        enet,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n    ])\n        \nmodel_enet.compile(\n    optimizer=tf.keras.optimizers.Adam(lr=0.0001),\n    loss = 'sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)\nmodel_enet.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_enet.fit(\n    get_training_dataset(),\n    steps_per_epoch = STEPS_PER_EPOCH,\n    epochs = EPOCHS,\n    callbacks = [lr_callback],\n    validation_data = get_validation_dataset(),\n    validation_steps=VALID_STEPS,\n    class_weight = weight_per_class\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_train_valid_curves(history.history['loss'], history.history['val_loss'], 'loss', 211) \nplot_train_valid_curves(history.history['sparse_categorical_accuracy'], \n                        history.history['val_sparse_categorical_accuracy'], 'accuracy', 212) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I don't like this approach because it looks very little like an ensemble of neural networks.\n\nI tried to do something similar with the Titanic competition, but, obviously, such mixing of results is much more suitable for neural networks than for, for example, some decision trees.","metadata":{}},{"cell_type":"code","source":"best_alpha=0.51\ntest_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = best_alpha*model_enet.predict(test_images_ds) + (1-best_alpha)*model_dens.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}