{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet ../input/kerasapplications\n!pip install --quiet ../input/efficientnet-git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set_style(\"whitegrid\")\n\nimport tensorflow as tf\nimport tensorflow_datasets as tfds\nimport keras\nfrom keras import backend as K\nfrom sklearn.model_selection import KFold\nfrom efficientnet import keras as efn\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport os, re, sys\nimport json\nimport math\nfrom functools import partial\n\ninputdir = \"/kaggle/input/cassava-leaf-disease-classification\"\nlistdir = os.listdir(inputdir)\nfor filename in listdir:\n    filepath = os.path.join(inputdir, filename)\n    print(filepath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is set. \n    # On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    print(\"tpu\")\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    DEVICE = \"TPU\"\nelse:\n    # default distribution strategy in Tensorflow. \n    # Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n    DEVICE = \"notTPU\"\n\nREPLICAS = strategy.num_replicas_in_sync\n\nprint(\"REPLICAS: \", REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_SIZE = 21397\nN_CLASS = 5\nN_SPLITS = 5\n\nIMAGE_SIZE = [256, 256]\nEPOCHS = 5\nBATCH_SIZE = 16 * REPLICAS \nAUG_BATCH = BATCH_SIZE\nSTEPS_PER_EPOCH = TRAIN_SIZE // BATCH_SIZE\nAUTO = tf.data.experimental.AUTOTUNE\n\nTRAIN_PATH = \"train_tfrecords/*.tfrec\"\nTEST_PATH = \"test_tfrecords/ld_test*.tfrec\"\nMAIN_PATH = \"/kaggle/input/cassava-leaf-disease-classification\"\n\nif DEVICE == \"TPU\":\n    MAIN_PATH = KaggleDatasets().get_gcs_path(\"cassava-leaf-disease-classification\")    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Disease Map"},{"metadata":{"trusted":true},"cell_type":"code","source":"json_path = \"/kaggle/input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\nwith open(json_path, \"r\") as file:\n    disease_map = json.load(file)\n\ndisease_map","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load TfRecords"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_filenames(path):\n    return tf.io.gfile.glob(path)\n\ndef load_records(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=-1)\n    return dataset\n\ntrain_filenames = get_filenames(os.path.join(MAIN_PATH, TRAIN_PATH))\ntest_filenames = get_filenames(os.path.join(MAIN_PATH, TEST_PATH))\n\ntrain_records = load_records(train_filenames)\ntest_records = load_records(test_filenames)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TFrecords features"},{"metadata":{},"cell_type":"markdown","source":"Train features: image, image_name, target\n\nTest features: image, image_name"},{"metadata":{},"cell_type":"markdown","source":"# Parse Tensors"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create features description to build their shape and type signature\nconfig = tf.io.FixedLenFeature\n\nfeature_train = {\"image\"     : config([], tf.string),\n                 \"image_name\": config([], tf.string),\n                 \"target\"    : config([], tf.int64),}\n\nfeature_test = {\"image_name\" : config([], tf.string),\n                \"image\"      : config([], tf.string),}\n\n\n# There is image feature inside the records, so we need to decode it\ndef decode_image(raw):\n    \"\"\"Decode parsed bytes string into jpeg format\"\"\"\n    \n    decoded = tf.io.decode_jpeg(raw) \n    image = tf.image.resize(decoded, size=IMAGE_SIZE)\n    image = tf.reshape(image, shape=(*IMAGE_SIZE, 3))\n    image = tf.cast(image, tf.float32) / 255. # normalize to 0..1 value\n    return image \n\n\n# Now we can parse our tfrecords\ndef read_tfrecords(example_single, features):\n    \"\"\"Parse raw bytes string from tfrec\"\"\"\n    \n    parsed = tf.io.parse_single_example(example_single, features)\n    \n    image = decode_image(parsed.get(\"image\"))\n    image_name = parsed.get(\"image_name\")\n    target = parsed.get(\"target\")\n    \n    # Train dataset\n    if features.get(\"target\"):\n        return image, target\n    \n    # Test dataset\n    return image_name, image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use functools.partial to set up default args for specific data\n\n\n# Default arg for train data\nparse_train = partial(read_tfrecords, features=feature_train)\n\n# Default arg for test data\nparse_test = partial(read_tfrecords, features=feature_test)\n\n\n# Map the previous functs to our dataset\ntrain_dataset = train_records.map(parse_train)\ntest_dataset = test_records.map(parse_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for x in train_dataset.take(1):\n    print(\"Image Shape {.shape}\".format(x[0].numpy()))\n    print(\"Target Shape {.shape}\".format(x[1].numpy()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA\nLeaf Categories:\n\n'0': 'Cassava Bacterial Blight (CBB)'      <br/>\n'1': 'Cassava Brown Streak Disease (CBSD)' <br/>\n'2': 'Cassava Green Mottle (CGM)'          <br/>\n'3': 'Cassava Mosaic Disease (CMD)'        <br/>\n'4': 'Healthy'                             <br/>"},{"metadata":{},"cell_type":"markdown","source":"### Class Distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read csv file\ntrain_csv = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n\nsns.countplot(x=\"label\", data=train_csv)\nplt.title(\"Label Distribution\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Filter dataset based on labels\nbacterial_blight = train_dataset.filter(lambda image, label: label == 0)\nbrown_disease    = train_dataset.filter(lambda image, label: label == 1)\ngreen_mottle     = train_dataset.filter(lambda image, label: label == 2)\nmosaic_disease   = train_dataset.filter(lambda image, label: label == 3)\nhealthy          = train_dataset.filter(lambda image, label: label == 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image(rows, cols, dataset):    \n    index = 1\n    plt.figure(figsize=(3 * cols, rows * 3))\n    for image, label in dataset.take(rows * cols):\n        plt.subplot(rows, cols, index)\n        plt.imshow(image.numpy())\n        plt.title(label.numpy())\n        plt.axis(\"off\")\n        index += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cassava Bacterial Blight"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(1, 5, bacterial_blight)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cassava Brown Streak Disease"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(1, 5, brown_disease)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cassava Green Mottle"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(1, 5, green_mottle)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cassava Mosaic Disease"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(1, 5, mosaic_disease)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Healthy"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image(1, 5, healthy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Augmentation"},{"metadata":{},"cell_type":"markdown","source":"### Rotation, Shift, Zoom, Shear"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rot_shift_zoom_shear(image, DIM = IMAGE_SIZE[0]):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n\n    XDIM = DIM % 2\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### CutMix\nfrom <a href=\"https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu\">here</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = N_CLASS\n    return image,tf.one_hot(label,CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = N_CLASS\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rotshift_aug(image, label):\n    prob = tf.random.uniform(shape=[], minval=0.0, maxval=1.0)\n    if prob <= 0.3:\n        image = rot_shift_zoom_shear(image)    \n    \n    label = tf.one_hot(label, N_CLASS)    \n    return tf.cast(image, tf.float32), label\n\n\ndef cutmix_aug(image, label):\n    prob = tf.random.uniform(shape=[], minval=0.0, maxval=1.0)\n    if prob <= 0.5: \n        image, label = cutmix(image, label,)\n    \n    return tf.cast(image, tf.float32), label    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmented Pictures"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Show augmented images\n# rows, cols = 2, 5\n# rows = min(rows, AUG_BATCH//cols)\n\n# augmented_element = train_dataset.batch(BATCH_SIZE).map(augmentation)\n\n# for (img,label) in augmented_element:\n#     plt.figure(figsize=(15,int(15*rows/cols)))\n#     for j in range(rows * cols):\n#         plt.subplot(rows, cols, j+1)\n#         plt.axis('off')\n#         plt.imshow(img[j,])\n#     plt.show();\n#     break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"def filter_layers(model, layer_name):\n    compiler = re.compile(r\"{name}\".format(name=layer_name))\n\n    for layer in model.layers:\n        if not compiler.search(layer.name):\n            layer.trainable = False # transfer learning\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dense_model = filter_layers(dense_net, \"conv5_block\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\"include_top\": False, \n          \"pooling\": \"avg\", \n          \"input_shape\": (*IMAGE_SIZE, 3),\n          \"weights\": None}\n\nloss = \"categorical_crossentropy\"\noptimizer = \"adam\"\nmetric = \"categorical_accuracy\"\n\nwith strategy.scope():\n    \n    # Load model\n    if DEVICE == \"TPU\":\n        dense_net = efn.EfficientNetB0(**params)\n        dense_net.load_weights(\"../input/pretrained-cnn-weights/efficientnet-b0_256_weights.h5\")\n    else:\n        dense_net = efn.EfficientNetB2(**params)\n        dense_net.load_weights(\"../input/pretrained-cnn-weights/efficientnet-b2_256_weights.h5\")\n    \n    # Build layers\n    input_image = keras.Input(shape=[*IMAGE_SIZE, 3])\n    dense_nets = dense_net(input_image)\n    dropout = keras.layers.Dropout(0.3)(dense_nets)\n    output = keras.layers.Dense(N_CLASS, activation=\"softmax\")(dropout)\n    \n    # Initialize Model\n    base_model = keras.models.Model(input_image, output)\n    \n    # Model Optimizer\n    optimizer = keras.optimizers.Adamax(learning_rate=1e-3)\n\n    base_model.compile(loss=loss, optimizer=optimizer, metrics=[metric])\n    \n    \n# Set initial weights\ninitial_weights = base_model.get_weights()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# KFold on Tf.Dataset\nInspired from : <a href=\"https://stackoverflow.com/questions/59669413/what-is-the-canonical-way-to-split-tf-dataset-into-test-and-validation-subsets\">here</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fold Train Dataset\nkfold = KFold(n_splits=N_SPLITS)\nsplits = list(kfold.split(np.arange(TRAIN_SIZE)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def kfold_dataset(dataset: tf.data.Dataset, fold_splits):\n    \"\"\"\n    Splits a dataset of type tf.data.Dataset into a training and validation dataset \n    using KFold splits.\n    \n    @param dataset: the input dataset to split.\n    @param fold_splits: splits produces from KFold .split() method.\n    @return: a tuple of two tf.data.Datasets as (training, validation)\n    \"\"\"\n    \n    train_fold, val_fold = fold_splits # train and val index\n    \n    val_min = np.min(val_fold) # val min index\n    val_max = np.max(val_fold) #     max index\n    \n    dataset = dataset.enumerate()\n    train_dataset = dataset.filter(lambda x, data: x <= val_min or x >= val_max)\n    validation_dataset = dataset.filter(lambda x, data: x >= val_min and x <= val_max)\n\n    # remove enumeration\n    train_dataset = train_dataset.map(lambda x, data: data)\n    validation_dataset = validation_dataset.map(lambda x, data: data)\n\n    return train_dataset, validation_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train steps\ntrain, val = splits[0]\nTRAIN_STEPS = len(train) // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, split in enumerate(splits):\n    \n    print(\"Fold {}\".format(fold))\n    model_name = \"model_fold_{}.h5\".format(fold)\n    weights_name = \"weights_fold_{}.h5\".format(fold)\n    \n    train_split, val_split = kfold_dataset(train_dataset, split)\n    \n    \n    train_input = (train_split.repeat()                                  # note: repeat then map\n                            .map(rotshift_aug, num_parallel_calls=AUTO)\n                            .batch(BATCH_SIZE)\n                            .map(cutmix_aug, num_parallel_calls=AUTO)\n                            .prefetch(AUTO))\n\n    val_input = val_split.map(onehot).batch(BATCH_SIZE)\n    \n    model_checkpoint = keras.callbacks.ModelCheckpoint(\n                                            filepath=model_name,\n                                            monitor=\"val_\" + metric,\n                                            save_best_only=True)\n    \n    base_model.fit(train_input, \n                     epochs=EPOCHS,\n                     steps_per_epoch=TRAIN_STEPS,\n                     validation_data=val_input,\n                     callbacks = [model_checkpoint])\n    \n    # Save weights\n    best_model = keras.models.load_model(model_name)\n    best_model.save_weights(weights_name)\n    \n    # Reset weights for every fold :D\n    base_model.set_weights(initial_weights)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test Dataset\ntest_input = test_dataset.batch(BATCH_SIZE)\n\n# Test Images and Names\ntest_images = test_input.map(lambda image_name, image: image)\ntest_names = test_input.map(lambda image_name, image: image_name).unbatch().batch(15000)\n\n# Define weights\nweights = []\n\n# Get all fold weights\nfor index in range(N_SPLITS):\n    base_model.load_weights(\"weights_fold_{}.h5\".format(index))\n    fold_weight = base_model.get_weights()\n    weights.append(fold_weight)\n    \n# Average of all weights\nw = []\nfor weight in zip(*weights):\n    w.append(np.mean(weight, axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set average weight\nbase_model.set_weights(w)\n\n# Predictions\nprobability = base_model.predict(test_images)\nprediction = np.argmax(probability, axis=1)\nimage_ids = next(iter(test_names)).numpy().astype(\"U\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n\nsubmission.loc[:, \"image_id\"] = image_ids\nsubmission.loc[:, \"label\"] = prediction\n\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Thing that work:\n- Using DenseNet201, DenseNet169 and EfficientNetB0\n- RotShift and CutMix Aug\n- KFold 5\n\n### Thing that didn't work:\n- GridMask Aug or any Blockout Aug\n\n### Models Note:\n- But in LB, DenseNet still win (0.86 LB) while efb0 (0.83 LB)\n- EfficientNetB0 perform better than DenseNet\n- DenseNet201 and DenseNet169 didn't differ so much (both perform 0.84 CV and 0.86 LB)\n\n\n### Optimizers Note:\n- AdaMax converge faster than Adam (less and equal than 5 epochs)\n- Adam performs well in this dataset (need more than 5 epochs to converge)\n- RMSProp performance is less than Adam (it stuck on suboptimal minima)\n- Adagrad here perform worse on start\n\n### Additional Note:\n- Both RotShift and CutMix Aug when performed individually, produce same result, but increase LB by 0.001 when perform simultaneously"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}