{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U efficientnet","execution_count":null,"outputs":[]},{"metadata":{"id":"s0Cwl0MdXDVR"},"cell_type":"markdown","source":"# Import libraries and set seeds"},{"metadata":{"id":"e1jtqEVI8Epv","trusted":true},"cell_type":"code","source":"import os, sys, json\nos.environ['PYTHONHASHSEED']='0'\nimport tensorflow as tf\ntf.random.set_seed(42)\nimport efficientnet.tfkeras as efn\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nnp.random.seed(42)\nimport tensorflow.keras as keras\nimport albumentations\nimport pickle\n# from google.colab.patches import cv2_imshow\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"id":"BH9ALEXyXIbI"},"cell_type":"markdown","source":"# EDA"},{"metadata":{"id":"fToNRvTlJftK","outputId":"9b4f1a5c-c746-4034-83e1-c9b45513afed","trusted":true},"cell_type":"code","source":"labels = json.loads(open('/kaggle/input/cassava-leaf-disease-classification/label_num_to_disease_map.json', 'r').read())\nprint(labels)","execution_count":null,"outputs":[]},{"metadata":{"id":"eT4gLG4xINDh","outputId":"75ce89e5-5359-46ac-9132-404e2ac1a379","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\nprint(train.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check the imbalance of no. of samples in each class"},{"metadata":{"id":"rQKdU2EFajv0","outputId":"d8f2722d-b309-48ae-f531-e5ead299d6f3","trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"eMk1oVZsXS2L"},"cell_type":"markdown","source":"## Estimate class weight scales to compensate for class imbalance\nInstead of resampling, I am going to scale the loss of each class by a respective class weight"},{"metadata":{"id":"xnkLCD1aa-TS","outputId":"07f32372-6625-4a9a-e546-dc4f4063c36a","trusted":true},"cell_type":"code","source":"weights = np.array([1087, 2189, 2386, 13158, 2577])\nprint(np.sum(weights))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## How?\nA quick way: Sklearn (https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_class_weight.html)\nClass weight = `total_samples/(samples_i * num_classes)`.\n\nFor 5 balanced classes, lets say total weight for all samples = 1.\n\nLet each class have x samples, and there are total 5x samples. (In our case, total=21397).\n\n$$  5x\\  ->\\ 1$$\n\n$$    x\\  ->\\ ? $$\n    \n$$  x\\  -> 1/5 (x\\ samples\\ scaled\\ by\\ 1/5)$$\n\nHere, 0th class has 1087 samples\n$$ X = \\sum\\limits_{i=1}^{5} {x} = 21397$$\n\n$$  x0\\  =\\ (1807*X)/(21397)$$\n\n$$  x\\  ->\\ 1/5$$\n\n$$ x0\\  ->\\ ? $$\n\n$$ x0\\  ->\\ 1807/(21397*5)$$\n\n(I have tried my best to explain it in a few lines, reading a proper aticle/paper is recommended)"},{"metadata":{"id":"b7w_qlcVbSwg","trusted":true},"cell_type":"code","source":"cl_w = []\nfor i in weights:\n    cl_w.append(np.sum(weights)/(5*i))","execution_count":null,"outputs":[]},{"metadata":{"id":"ILfDMGPsbnA4","outputId":"6743d567-828f-4450-bfcf-8bef08fd9b05","trusted":true},"cell_type":"code","source":"cl_w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"norm_cw = np.sqrt(np.array(cl_w))","execution_count":null,"outputs":[]},{"metadata":{"id":"-Sz7cQaCa4dr","trusted":true},"cell_type":"code","source":"class_weights = {0:norm_cw[0], 1:norm_cw[1], 2:norm_cw[2], 3:norm_cw[3], 4:norm_cw[4]}","execution_count":null,"outputs":[]},{"metadata":{"id":"DKr2wLpMhCZ_","outputId":"512ece70-c8dc-4738-d78f-f39c01402ba3","trusted":true},"cell_type":"code","source":"class_weights","execution_count":null,"outputs":[]},{"metadata":{"id":"7XaLoaWmXg3y"},"cell_type":"markdown","source":"## Visualization\nPlot 5 samples of each class"},{"metadata":{"id":"8B8R2fSFLVNc","trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(20,20))\ncount = 0\nimg_classes = {0:5, 1:5, 2:5, 3:5, 4:5}\nfor idx, i in enumerate(list(os.listdir('/kaggle/input/cassava-leaf-disease-classification/train_images/'))):\n    img_class = train.loc[train['image_id']==i]['label'].tolist()[0]\n    if img_classes[img_class] > 0:\n        img_classes[img_class] -= 1\n    else:\n        continue\n    ax = fig.add_subplot(5, 5, count+1)\n    ax.set_title(\"Class = \"+str(train.loc[train['image_id']==i]['label'].tolist()[0]))\n    img = cv2.imread('/kaggle/input/cassava-leaf-disease-classification/train_images/'+i)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    plt.imshow(img)\n    count += 1\n    if count == 25:\n        break","execution_count":null,"outputs":[]},{"metadata":{"id":"5oWHNTaZ3-NR"},"cell_type":"markdown","source":"# Prepare dataset\nA very simple tutorial from keras on how to use tfrecords: https://keras.io/examples/keras_recipes/tfrecord/"},{"metadata":{"id":"8HPBabR6ii8F","outputId":"f49c5516-2794-4002-f787-9354c4f18cf8","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"id":"yT2DGOk5ktmd","trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path()\nprint(GCS_PATH)\nBATCH_SIZE = 128\nIMAGE_SIZE = [512, 680]","execution_count":null,"outputs":[]},{"metadata":{"id":"DtHTkC1GipZf","trusted":true},"cell_type":"code","source":"tfrecords = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/ld_train*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"id":"bCinI7kC4B7H","trusted":true},"cell_type":"code","source":"FILENAMES = tfrecords\nsplit_ind = int(0.9 * len(FILENAMES))\nTRAINING_FILENAMES, VALID_FILENAMES = FILENAMES[:split_ind], FILENAMES[split_ind:]","execution_count":null,"outputs":[]},{"metadata":{"id":"mcdTMUEXjAIb","outputId":"e0eb8607-7675-4de5-e3d0-7d2cb61f35e5","trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"id":"pgKKBhnvlQ2t","trusted":true},"cell_type":"code","source":"def decode_image(image):\n    \"\"\"\n        decode/read image and and cast to fp32\n    \"\"\"\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    return image\n\ndef train_preprocess(image, label):\n    \"\"\"\n        Data augmentations for training\n    \"\"\"\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_brightness(image, max_delta=0.2)\n#     image = tf.image.random_contrast(image, lower=0.1, upper=0.2)\n#     image = tf.image.random_saturation(image, 2, 4)\n    image = tf.image.random_jpeg_quality(image, 90, 100)\n    image = tf.image.per_image_standardization(image)\n    image = tf.image.random_crop(image, [500, 500, 3])\n    image = tf.image.resize(image, IMAGE_SIZE, method=tf.image.ResizeMethod.BILINEAR, preserve_aspect_ratio=False)\n    return image, label\n\ndef val_preprocess(image, label):\n    \"\"\"\n        Only image standardization for validation data\n    \"\"\"\n    image = tf.image.per_image_standardization(image)\n    image = tf.image.resize(image, IMAGE_SIZE, method=tf.image.ResizeMethod.BILINEAR, preserve_aspect_ratio=False)\n    \n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"id":"WdGU80xdkJxK","trusted":true},"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = (\n        {\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"target\": tf.io.FixedLenFeature([], tf.int64),\n        }\n        if labeled\n        else {\"image\": tf.io.FixedLenFeature([], tf.string),}\n    )\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example[\"image\"])\n    if labeled:\n        label = tf.cast(example[\"target\"], tf.int32)\n        # One hot encode label\n        label = tf.one_hot(indices=label, depth=5, on_value=1, off_value=0)\n        return image, label\n    return image","execution_count":null,"outputs":[]},{"metadata":{"id":"wF3vtGAg_1CN","trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True, transforms=None):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames)  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order)  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n    return dataset\n\ndef get_dataset(filenames, labeled=True, transforms=None):\n    dataset = load_dataset(filenames, labeled=labeled, transforms=transforms)\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    dataset = dataset.map(transforms, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"id":"CNaUTLVmAIx4","trusted":true},"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, labeled=True, transforms=train_preprocess)\nvalid_dataset = get_dataset(VALID_FILENAMES, labeled=True, transforms=val_preprocess)\nfull_dataset = get_dataset(FILENAMES, labeled=True, transforms=val_preprocess)","execution_count":null,"outputs":[]},{"metadata":{"id":"qnME3DgAh9gP","outputId":"77ad7a99-ec96-452f-c341-d71fa429ee4e","trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{"id":"H3b6DNrG8eKh","trusted":true},"cell_type":"code","source":"# lr scheduler\ninitial_learning_rate = 0.00025\nlr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate, decay_steps=4, decay_rate=0.96, staircase=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir(tf.keras.applications)","execution_count":null,"outputs":[]},{"metadata":{"id":"VqZ7iUdZmKJv","trusted":true},"cell_type":"code","source":"def make_model():\n    base_model = efn.EfficientNetB5(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False, weights='noisy-student'\n    )\n\n    base_model.trainable = True\n\n    inputs = tf.keras.layers.Input([*IMAGE_SIZE, 3])\n    # x = tf.keras.applications.densenet.preprocess_input(inputs)\n    x = base_model(inputs)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(128, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    outputs = tf.keras.layers.Dense(5, activation=\"softmax\")(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n    \n    # Use lr_scheduler as input to the optimizer\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),\n        loss=keras.losses.CategoricalCrossentropy(),\n        metrics=['accuracy']\n    )\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define callbacks\nFor early stopping and saving checkpoints. I am not using LR on plateau as I have used a scheduler instead (I am not training for a lot of epochs and the network is pretrained)"},{"metadata":{"id":"YwRHhtJzng7N","trusted":true},"cell_type":"code","source":"cp_path = \"/kaggle/working/cp2_efn7_latest.h5\"\ncallbacks = [keras.callbacks.ModelCheckpoint(cp_path, verbose=1, save_best_only=True, monitor='val_loss'),\n             keras.callbacks.EarlyStopping(patience=3, verbose=True)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = make_model()","execution_count":null,"outputs":[]},{"metadata":{"id":"ByK7EjNnynXZ","outputId":"664ba0b7-3c98-49dd-97a6-5f212cfc43fe","trusted":true},"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    epochs=15,\n    validation_data=valid_dataset,\n    callbacks=callbacks,\n    class_weight=class_weights,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One last epoch with all the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = keras.models.load_model('/kaggle/working/cp2_efn7_latest.h5')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),\n        loss=keras.losses.CategoricalCrossentropy(),\n        metrics=['accuracy']\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    epochs=1,\n    validation_data=valid_dataset,\n    callbacks=callbacks,\n    class_weight=class_weights,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('final_eb5.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Since final submission does not allow TPU's, the inference script will be separate"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}