{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/cassava-leaf-disease-classification/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Much of the code used here is based on: https://keras.io/examples/keras_recipes/tfrecord/\n#More code taken from my work on the MARCO crystallization image dataset\n\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom mpl_toolkits.axes_grid1 import ImageGrid\n\nimport glob\nimport os\n\nfrom PIL import Image\n\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, Conv3D, MaxPooling2D, Flatten, Dense, Activation, Dropout\nfrom keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop\n\nfrom functools import partial\n\n# Tuning and training params:\nBATCH_SIZE = 1\nIMAGE_SIZE = [512, 512]\nepochs = 18\n\ncsv_df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\ncsv_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Our data is unbalanced. We'll have to remedy that or all the predictions will be \"3\"\nsns.distplot(csv_df['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Getting weights to account for unbalanced data\n# weight = 1/(number of images in class)*(size of dataset)/(number of classes)\nweights = [(1/csv_df['label'].value_counts()[n])*csv_df['label'].shape[0]/5 for n in csv_df['label'].unique()]\nweights = dict(zip(list(range(0,5)),weights))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"![](http://)The data here provided is also provided in tfrec format which speeds up training a good deal."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '/kaggle/input/cassava-leaf-disease-classification/train_tfrecords/*'\ntest_dir = '/kaggle/input/cassava-leaf-disease-classification/test_tfrecords/*'\n\ntrain_list = glob.glob(train_dir)\ntest_list = glob.glob(test_dir)\nprint(\"train: \" + str(len(train_list)) + \"\\ntest: \" + str(len(test_list)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building a data generator\n\nTo get tfrecords into a neural-network for training we need to first build a data-generator to make batches of images (which fit in our RAM)\nWe need a few steps to get there.\n1. What features are in the TFRecords? We need to unpack them and see what features are in there.\n2. We need a getter to give us shuffled training batches that don't overwhelm the ram.\n3. The getter needs a map function to read the features out of the dataset.\n4. The map function needs an image decoder to do our image manipulations for us.\n5. The image decoder should have an option to perform augmentation operations."},{"metadata":{"trusted":true},"cell_type":"code","source":"#To get some idea what features are in the TFRecords, we need to unpack them.\n# Found this code on stackexchange posted by user jdehesa\n\ndef list_record_features(tfrecords_path):\n    # Dict of extracted feature information\n    features = {}\n    # Iterate records\n    for rec in tf.data.TFRecordDataset([str(tfrecords_path)]):\n        # Get record bytes\n        example_bytes = rec.numpy()\n        # Parse example protobuf message\n        example = tf.train.Example()\n        example.ParseFromString(example_bytes)\n        # Iterate example features\n        for key, value in example.features.feature.items():\n            # Kind of data in the feature\n            kind = value.WhichOneof('kind')\n            # Size of data in the feature\n            size = len(getattr(value, kind).value)\n            # Check if feature was seen before\n            if key in features:\n                # Check if values match, use None otherwise\n                kind2, size2 = features[key]\n                if kind != kind2:\n                    kind = None\n                if size != size2:\n                    size = None\n            # Save feature data\n            features[key] = (kind, size)\n    return features\n\nrecord_features = list_record_features(glob.glob(train_dir+\"*\")[0])\nprint(record_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_description = {\n    \"image\" : tf.io.VarLenFeature(tf.string), #image jpg encoded\n    \"image_name\" : tf.io.FixedLenFeature([], tf.string), #image unique identifier\n    \"target\" : tf.io.FixedLenFeature([], tf.int64), #Target class\n}\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\n# I originally got this code from https://keras.io/examples/keras_recipes/tfrecord/ - Light customizations as needed.\n\ndef get_dataset(filenames, labeled=True):\n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.shuffle(buffer_size = 2048, seed = 42)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset\n\n\ndef load_dataset(filenames, labeled=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.with_options(ignore_order) # again, faster.\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    return dataset\n\n\ndef read_tfrecord(example, labeled=True):\n    if labeled:\n        tfrecord_format = (\n            {\n                \"image\": tf.io.VarLenFeature(tf.string),\n                \"target\": tf.io.FixedLenFeature([], tf.int64),\n                \"image_name\": tf.io.FixedLenFeature([], tf.string)\n            })\n    else:\n        tfrecord_format = ({\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"image_name\": tf.io.FixedLenFeature([], tf.string)\n        })\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example[\"image\"], labeled)\n    name = example[\"image_name\"]\n    if labeled:\n        label = tf.cast(example[\"target\"], tf.int32)\n        label = tf.one_hot(label, depth = 5)\n        return image, label\n    else:\n        return image, name\n\n\ndef decode_image(image, labeled):\n    if labeled:\n        image = tf.image.decode_jpeg(image.values[0], channels=3)\n        image = augment(image)\n    else:\n        image = tf.image.decode_jpeg(image, channels = 3)\n    image = tf.cast(image, tf.float32)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef augment(image):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_brightness(image, max_delta=0.5)\n    image = tf.image.random_crop(image, size=[*IMAGE_SIZE, 3])\n    #TODO: Add augmentation. MixUp and CutUp from https://www.kaggle.com/markwijkhuizen/tf-efficientnetb4-mixup-and-cutmix-cv-0-90\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lrfn(epoch, bs=BATCH_SIZE, EPOCHS=epochs):\n    # Config\n    LR_START = 1e-6\n    LR_MAX = 2e-4\n    LR_FINAL = 1e-6\n    LR_RAMPUP_EPOCHS = epochs/10\n    LR_SUSTAIN_EPOCHS = 0\n    DECAY_EPOCHS = epochs  - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1\n    LR_EXP_DECAY = (LR_FINAL / LR_MAX) ** (1 / (EPOCHS - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS - 1))\n\n    if epoch < LR_RAMPUP_EPOCHS: # exponential warmup\n        lr = LR_START + (LR_MAX + LR_START) * (epoch / LR_RAMPUP_EPOCHS) ** 2.5\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS: # sustain lr\n        lr = LR_MAX\n    else: # cosine decay\n        epoch_diff = epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS\n        decay_factor = (epoch_diff / (DECAY_EPOCHS+0.01)) * 3.14\n        decay_factor= (tf.math.cos(decay_factor).numpy() + 1) / 2        \n        lr = LR_FINAL + (LR_MAX - LR_FINAL) * decay_factor\n\n    return lr\n\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"cassava_model_checkpoint.h5\", save_best_only=True\n)\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(\n    patience=10, restore_best_weights=True\n)\n\nlr_cb = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=1)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ready for training\nBatch size - we'll do 3, it's what fits in ram. If we decrease the image resolution we'll turn that up."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Tuning and training params:\nBATCH_SIZE = 3\nIMAGE_SIZE = [299, 299]\nepochs = 10\n\nprint(\"Train Files: \", len(train_list))\nprint(\"Test Files: \", len(test_list))\n\ntrain_dataset = get_dataset(train_list[1])\nvalid_dataset = get_dataset(train_list[0])\ntest_dataset = get_dataset(test_list, labeled=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inception V4 \nIt's not added to keras yet so I added it here as a utility script. IV4 may not be the best choice for this competition since it's very computationally intensive but it has the best balance of time-to-train and performance across multiple tasks as compared to other model architectures so I'm using it here."},{"metadata":{"trusted":true},"cell_type":"code","source":"from inceptionv4 import create_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = create_model(\n    num_classes=5,\n)\n\nbase_model.trainable = True\ninputs = tf.keras.Input(shape=([*IMAGE_SIZE, 3]))\nx = base_model(inputs, training = True)\n#x = tf.keras.layers.Dropout(0.4)(x)\n#x = tf.keras.layers.Dense(2000, activation='relu')(x)\n#x = tf.keras.layers.Dense(500, activation='relu')(x)\noutputs = tf.keras.layers.Dense(5, activation='sigmoid')(x)\nmodel = tf.keras.Model(inputs, outputs)\n\nmodel.compile(optimizer='adam', loss=tf.keras.losses.categorical_crossentropy,\n              metrics=tf.keras.metrics.categorical_accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 10\nmodel.fit(\n    train_dataset,\n    #validation_data = valid_dataset,\n    batch_size = 8,\n    epochs = epochs,\n    callbacks = [lr_cb, checkpoint_cb, early_stopping_cb],\n    class_weight = weights\n)\nmodel.save(\"./Cassava-Model.h5\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(valid_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_pred.argmax(axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = []\nids = []\nfor n in test_dataset:\n    name = (str(n[1].numpy()[0]).replace('b', \"\").replace(\"'\", \"\"))\n    pred = model.predict(n[0]).argmax()\n    ids.append(name)\n    labels.append(pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = pd.DataFrame.from_dict({\"image_id\":ids, \"label\":labels})\npreds.to_csv(\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# It's still overfitting and I'm out of time.\nAs much as I hate to give up, it's time to turn in.\nSee you next kernel."},{"metadata":{"trusted":true},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}