{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras import layers\nimport tensorflow_hub as hub\nimport pandas as pd\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport cv2\nimport csv","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:00.365857Z","iopub.execute_input":"2021-11-12T12:28:00.366553Z","iopub.status.idle":"2021-11-12T12:28:05.50709Z","shell.execute_reply.started":"2021-11-12T12:28:00.366515Z","shell.execute_reply":"2021-11-12T12:28:05.506248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:05.508895Z","iopub.execute_input":"2021-11-12T12:28:05.509184Z","iopub.status.idle":"2021-11-12T12:28:05.525573Z","shell.execute_reply.started":"2021-11-12T12:28:05.509154Z","shell.execute_reply":"2021-11-12T12:28:05.524644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:05.528418Z","iopub.execute_input":"2021-11-12T12:28:05.528667Z","iopub.status.idle":"2021-11-12T12:28:05.579487Z","shell.execute_reply.started":"2021-11-12T12:28:05.528634Z","shell.execute_reply":"2021-11-12T12:28:05.578864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df.loc[:20896]\nval_df = df.loc[20897:].reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:05.581431Z","iopub.execute_input":"2021-11-12T12:28:05.581905Z","iopub.status.idle":"2021-11-12T12:28:05.586663Z","shell.execute_reply.started":"2021-11-12T12:28:05.58187Z","shell.execute_reply":"2021-11-12T12:28:05.585877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_id'] = '../input/cassava-leaf-disease-classification/train_images'+os.sep + train_df['image_id']\nval_df['image_id'] = '../input/cassava-leaf-disease-classification/train_images'+os.sep + val_df['image_id']","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:05.588141Z","iopub.execute_input":"2021-11-12T12:28:05.588425Z","iopub.status.idle":"2021-11-12T12:28:05.602609Z","shell.execute_reply.started":"2021-11-12T12:28:05.588388Z","shell.execute_reply":"2021-11-12T12:28:05.601766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_ds = tf.data.Dataset.from_tensor_slices((train_df['image_id'],train_df['label']))\nval_files_ds = tf.data.Dataset.from_tensor_slices((val_df['image_id'],val_df['label']))","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:05.603957Z","iopub.execute_input":"2021-11-12T12:28:05.604396Z","iopub.status.idle":"2021-11-12T12:28:07.94491Z","shell.execute_reply.started":"2021-11-12T12:28:05.604357Z","shell.execute_reply":"2021-11-12T12:28:07.944101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = 5\nEPOCHS = 30\nBATCH_SIZE = 10\nAUG_BATCH = BATCH_SIZE\nIMAGE_SIZE = (224,224)\nNUM_TRAINING_IMAGES = len(train_df)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:28:07.946269Z","iopub.execute_input":"2021-11-12T12:28:07.948174Z","iopub.status.idle":"2021-11-12T12:28:07.953737Z","shell.execute_reply.started":"2021-11-12T12:28:07.94813Z","shell.execute_reply":"2021-11-12T12:28:07.952667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def handle_filenames(filename,label):\n    image = tf.io.read_file(filename)\n    image = tf.io.decode_jpeg(image,channels = 3)\n    image = tf.cast(image,tf.float32)/255.0\n    image = tf.image.resize(image,size = (224,224))\n    return image,tf.one_hot(label,CLASSES)\n\n\ntrain_ds = train_files_ds.map(handle_filenames).repeat().batch(BATCH_SIZE).prefetch(AUTO)\nval_ds = val_files_ds.map(handle_filenames).batch(BATCH_SIZE).prefetch(AUTO)","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:30:03.359794Z","iopub.execute_input":"2021-11-12T12:30:03.360667Z","iopub.status.idle":"2021-11-12T12:30:03.451988Z","shell.execute_reply.started":"2021-11-12T12:30:03.360619Z","shell.execute_reply":"2021-11-12T12:30:03.451127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1)\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:30:08.075695Z","iopub.execute_input":"2021-11-12T12:30:08.07623Z","iopub.status.idle":"2021-11-12T12:30:08.089637Z","shell.execute_reply.started":"2021-11-12T12:30:08.076191Z","shell.execute_reply":"2021-11-12T12:30:08.088979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:33:03.203818Z","iopub.execute_input":"2021-11-12T12:33:03.20454Z","iopub.status.idle":"2021-11-12T12:33:03.214586Z","shell.execute_reply.started":"2021-11-12T12:33:03.204502Z","shell.execute_reply":"2021-11-12T12:33:03.21381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = train_ds.unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(cutmix)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:30:10.771512Z","iopub.execute_input":"2021-11-12T12:30:10.771812Z","iopub.status.idle":"2021-11-12T12:30:12.5434Z","shell.execute_reply.started":"2021-11-12T12:30:10.771778Z","shell.execute_reply":"2021-11-12T12:30:12.542489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = train_ds.unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(mixup)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:34:00.424602Z","iopub.execute_input":"2021-11-12T12:34:00.424877Z","iopub.status.idle":"2021-11-12T12:34:01.750547Z","shell.execute_reply.started":"2021-11-12T12:34:00.424843Z","shell.execute_reply":"2021-11-12T12:34:01.74833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(image,label):\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 5\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:35:43.63103Z","iopub.execute_input":"2021-11-12T12:35:43.631659Z","iopub.status.idle":"2021-11-12T12:35:43.63963Z","shell.execute_reply.started":"2021-11-12T12:35:43.631618Z","shell.execute_reply":"2021-11-12T12:35:43.638735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 6; col = 4;\nrow = min(row,AUG_BATCH//col)\nall_elements = train_ds.unbatch()\naugmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\nfor (img,label) in augmented_element:\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:36:37.291655Z","iopub.execute_input":"2021-11-12T12:36:37.291969Z","iopub.status.idle":"2021-11-12T12:36:40.13136Z","shell.execute_reply.started":"2021-11-12T12:36:37.291908Z","shell.execute_reply":"2021-11-12T12:36:40.130447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess = keras.Sequential([\n    layers.RandomFlip(),\n    layers.RandomRotation(0.2),\n    layers.RandomZoom((-0.2, 0)),\n    layers.RandomContrast((0.2,0.2))\n])\n\n#vit_backbone = hub.KerasLayer(\"https://tfhub.dev/sayakpaul/vit_b8_fe/1\", trainable=False) \nbackbone = keras.applications.EfficientNetB3(\n    include_top = False,\n    weights = '../input/efficientnetb3-notop/efficientnetb3_notop.h5',\n    input_shape = (224,224,3)\n)\nresnet_preprocess = keras.applications.efficientnet.preprocess_input\nbackbone.trainable = True","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:39:17.613747Z","iopub.execute_input":"2021-11-12T12:39:17.614023Z","iopub.status.idle":"2021-11-12T12:39:21.231846Z","shell.execute_reply.started":"2021-11-12T12:39:17.613991Z","shell.execute_reply":"2021-11-12T12:39:21.231058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decay_steps = int(round(len(train_df)/10))*10\ncosine_decay = keras.experimental.CosineDecay(initial_learning_rate=1e-4, decay_steps=decay_steps, alpha=0.3)","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:39:25.748636Z","iopub.execute_input":"2021-11-12T12:39:25.748891Z","iopub.status.idle":"2021-11-12T12:39:25.753461Z","shell.execute_reply.started":"2021-11-12T12:39:25.748863Z","shell.execute_reply":"2021-11-12T12:39:25.752332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    inp = keras.Input(shape = (224,224,3))\n    x = preprocess(inp)\n    x = resnet_preprocess(x)\n    x = backbone(x)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.Dense(5,activation = 'softmax',dtype = tf.float32)(x)\n    model = keras.Model(inp,x)\n    model.compile(\n        loss = 'categorical_crossentropy',\n        optimizer = keras.optimizers.Adam(cosine_decay),\n        metrics = ['categorical_accuracy']\n    )\n","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:39:28.635982Z","iopub.execute_input":"2021-11-12T12:39:28.636683Z","iopub.status.idle":"2021-11-12T12:39:30.074478Z","shell.execute_reply.started":"2021-11-12T12:39:28.636646Z","shell.execute_reply":"2021-11-12T12:39:30.073711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 0.0001\nLR_MAX = 0.001 * strategy.num_replicas_in_sync\nLR_MIN = 0.0001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T12:38:44.481104Z","iopub.execute_input":"2021-11-08T12:38:44.483372Z","iopub.status.idle":"2021-11-08T12:38:44.493737Z","shell.execute_reply.started":"2021-11-08T12:38:44.483309Z","shell.execute_reply":"2021-11-08T12:38:44.492591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aug_train = train_ds.unbatch()\naug_train = aug_train.repeat().batch(AUG_BATCH).map(transform)","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:41:39.195601Z","iopub.execute_input":"2021-11-12T12:41:39.195864Z","iopub.status.idle":"2021-11-12T12:41:40.165653Z","shell.execute_reply.started":"2021-11-12T12:41:39.195834Z","shell.execute_reply":"2021-11-12T12:41:40.164953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', \n    patience=5,\n    restore_best_weights = True\n)\nmodel.fit(aug_train,\n          validation_data = val_ds,\n          epochs = 30,\n          steps_per_epoch = STEPS_PER_EPOCH,\n          callbacks = [early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-11-12T12:42:20.921052Z","iopub.execute_input":"2021-11-12T12:42:20.921382Z","iopub.status.idle":"2021-11-12T12:48:36.389647Z","shell.execute_reply.started":"2021-11-12T12:42:20.921343Z","shell.execute_reply":"2021-11-12T12:48:36.388587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('aug_model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-11-08T12:41:17.775038Z","iopub.execute_input":"2021-11-08T12:41:17.775881Z","iopub.status.idle":"2021-11-08T12:41:18.407966Z","shell.execute_reply.started":"2021-11-08T12:41:17.775827Z","shell.execute_reply":"2021-11-08T12:41:18.407139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = os.listdir(\"../input/cassava-leaf-disease-classification/test_images\")\nparent = \"../input/cassava-leaf-disease-classification/test_images/\"\npredictions = []\npredictions.append([\"image_id\", \"label\"])\nfor i in l :\n    child = parent + i\n    img = cv2.imread(child)\n    img = cv2.resize(img, (224, 224))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = tf.keras.applications.efficientnet.preprocess_input(img)\n    img = img.reshape((1, 224, 224, 3))\n    pred = model.predict(img)\n    pred = pred.reshape((5,))\n    print(np.argmax(pred))\n    del img\n    predictions.append([i, str(np.argmax(pred))])\n    del pred\n\n\nwith open('submission.csv', 'w', newline='') as file:\n    writer = csv.writer(file)\n    writer.writerows(predictions)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T12:39:54.918518Z","iopub.execute_input":"2021-11-08T12:39:54.919008Z","iopub.status.idle":"2021-11-08T12:39:54.998894Z","shell.execute_reply.started":"2021-11-08T12:39:54.918957Z","shell.execute_reply":"2021-11-08T12:39:54.998211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-08T12:40:42.057202Z","iopub.execute_input":"2021-11-08T12:40:42.057686Z","iopub.status.idle":"2021-11-08T12:40:42.069805Z","shell.execute_reply.started":"2021-11-08T12:40:42.057647Z","shell.execute_reply":"2021-11-08T12:40:42.069053Z"},"trusted":true},"execution_count":null,"outputs":[]}]}