{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference Notebook is found here\n[Inference Notebook](https://www.kaggle.com/shubham219/experiment-with-models-in-keras-inference)","metadata":{}},{"cell_type":"markdown","source":"# Importing All The Required Liraries","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as tfl\nimport shutil\nfrom functools import partial\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split \nfrom sklearn.utils import class_weight\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import KFold\n\n\nprint(\"Tensorflow version -\",tf.__version__)\nprint(\"Python version\")\n!python --version","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-05T02:42:55.206963Z","iopub.execute_input":"2023-07-05T02:42:55.207332Z","iopub.status.idle":"2023-07-05T02:43:37.974264Z","shell.execute_reply.started":"2023-07-05T02:42:55.207304Z","shell.execute_reply":"2023-07-05T02:43:37.973210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU Config","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:37.976212Z","iopub.execute_input":"2023-07-05T02:43:37.977028Z","iopub.status.idle":"2023-07-05T02:43:46.466225Z","shell.execute_reply.started":"2023-07-05T02:43:37.976998Z","shell.execute_reply":"2023-07-05T02:43:46.465272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Variables","metadata":{}},{"cell_type":"code","source":"EPOCHS = 100\nIMAGE_SIZE = (256, 256)\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nBUFFER_SIZE = 32\nFOLD = 5\nSEED = (2, 3)\nBATCH_SIZE = 16*strategy.num_replicas_in_sync\n\nloss_list = []\nacc_list = []\nval_acc_list = []\nval_loss_list = []","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-07-05T02:43:46.467308Z","iopub.execute_input":"2023-07-05T02:43:46.467580Z","iopub.status.idle":"2023-07-05T02:43:46.473372Z","shell.execute_reply.started":"2023-07-05T02:43:46.467556Z","shell.execute_reply":"2023-07-05T02:43:46.472576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Metadata","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\n\nwith open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json') as file:\n    text = file.read()\nprint(text)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.475239Z","iopub.execute_input":"2023-07-05T02:43:46.475494Z","iopub.status.idle":"2023-07-05T02:43:46.523098Z","shell.execute_reply.started":"2023-07-05T02:43:46.475472Z","shell.execute_reply":"2023-07-05T02:43:46.522293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution Of Classes\nDataset is very imbalanced","metadata":{}},{"cell_type":"code","source":"figure = plt.figure(figsize=(8,4))\n(data['label'].value_counts()/len(data)*100).plot(kind='bar')\nplt.title(\"Distribution of Classes\")\nplt.ylabel('% count of classes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.524162Z","iopub.execute_input":"2023-07-05T02:43:46.524457Z","iopub.status.idle":"2023-07-05T02:43:46.853232Z","shell.execute_reply.started":"2023-07-05T02:43:46.524432Z","shell.execute_reply":"2023-07-05T02:43:46.852405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tot_len = len(data)\nclass_weights = data['label'].value_counts()\nclass_weights = tot_len/class_weights\nclass_weights = class_weights.to_dict()\nclass_weights = {k:v for k,v in sorted(class_weights.items()) }\nclass_weights","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.854256Z","iopub.execute_input":"2023-07-05T02:43:46.854522Z","iopub.status.idle":"2023-07-05T02:43:46.862914Z","shell.execute_reply.started":"2023-07-05T02:43:46.854499Z","shell.execute_reply":"2023-07-05T02:43:46.861930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 5\n    return image,tf.one_hot(label,CLASSES)\n\ndef augmentation(image, label):\n    aug = tf.keras.models.Sequential()\n    aug.add(tfl.RandomFlip('horizontal'))\n    aug.add(tfl.RandomRotation(0.3))\n    image = aug(image)\n    return image, label\n\ndef resnet_preprocess(image, label):\n    \n    preprocess = tf.keras.applications.resnet50.preprocess_input\n    image = preprocess(image)\n    \n    return image, label\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.864142Z","iopub.execute_input":"2023-07-05T02:43:46.864462Z","iopub.status.idle":"2023-07-05T02:43:46.873989Z","shell.execute_reply.started":"2023-07-05T02:43:46.864434Z","shell.execute_reply":"2023-07-05T02:43:46.873143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading TF Records - Helper Function","metadata":{}},{"cell_type":"code","source":"# Decode the data\ndef decode_image(image):\n    print(\"Reading Image\")\n    image = tf.image.decode_jpeg(image,channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    \n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\ndef load_dataset(filenames, labeled=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(\n        filenames\n    )  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(\n        ignore_order\n    )  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(\n        partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE\n    )\n    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n    return dataset\n\ndef get_training_dataset(filenames, labeled=True):\n    \n    dataset = load_dataset(filenames, labeled=labeled)\n   \n    dataset = dataset.map(augmentation, num_parallel_calls = AUTOTUNE)\n    dataset = dataset.map(resnet_preprocess, num_parallel_calls = AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n   \n    return dataset\n\ndef get_validation_dataset(filenames, labeled=True):\n    \n    dataset = load_dataset(filenames, labeled=labeled)\n    dataset = dataset.map(resnet_preprocess, num_parallel_calls = AUTOTUNE)\n    \n    dataset = dataset.batch(BATCH_SIZE)\n    \n#     dataset = dataset.cache()\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n   \n    return dataset","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.875063Z","iopub.execute_input":"2023-07-05T02:43:46.875347Z","iopub.status.idle":"2023-07-05T02:43:46.891550Z","shell.execute_reply.started":"2023-07-05T02:43:46.875313Z","shell.execute_reply":"2023-07-05T02:43:46.890690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '/kaggle/input/cassava-leaf-disease-classification/train_tfrecords'\ntest_dir = '/kaggle/input/cassava-leaf-disease-classification/test_tfrecords'\n\nfile_names =  os.listdir(train_dir)\nfile_names = [os.path.join(train_dir, i) for i in file_names]\n\n# Splitting the data for validation set\ntrain_file_names = file_names[0:-3]\nvalidation_file_names = file_names[-3:]\n\nprint(len(file_names))\nprint(len(train_file_names))\nprint(len(validation_file_names))\n\n# test_files =  os.listdir(test_dir)\n# test_files = [os.path.join(test_dir, i) for i in test_files]","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.892597Z","iopub.execute_input":"2023-07-05T02:43:46.892895Z","iopub.status.idle":"2023-07-05T02:43:46.919975Z","shell.execute_reply.started":"2023-07-05T02:43:46.892861Z","shell.execute_reply":"2023-07-05T02:43:46.919096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Gettig the data in tensor flow dataset format\ntrain_ds = get_training_dataset(train_file_names)\nvalidation_ds = get_validation_dataset(validation_file_names)\n\n# Counting the Number Of Files\ntrain_images_cnt = sum([int(i) for i in re.findall('train\\d+-(\\d+)', str(train_file_names))])\n\nvalidation_images_cnt = sum([int(i) for i in re.findall('train\\d+-(\\d+)', str(validation_file_names))])\n\nprint(\"# Training Images: \", train_images_cnt)\nprint(\"# Validation Images: \", validation_images_cnt)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:46.923172Z","iopub.execute_input":"2023-07-05T02:43:46.923455Z","iopub.status.idle":"2023-07-05T02:43:47.604005Z","shell.execute_reply.started":"2023-07-05T02:43:46.923431Z","shell.execute_reply":"2023-07-05T02:43:47.602725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for img, label in train_ds.take(1):\n#     print(img[0])\n    print(\"Image Size: \",img.shape)\n    print(\"Labels :\", label.shape)\n    print(label)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:47.605310Z","iopub.execute_input":"2023-07-05T02:43:47.605597Z","iopub.status.idle":"2023-07-05T02:43:51.642578Z","shell.execute_reply.started":"2023-07-05T02:43:47.605571Z","shell.execute_reply":"2023-07-05T02:43:51.641323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:51.644049Z","iopub.execute_input":"2023-07-05T02:43:51.644374Z","iopub.status.idle":"2023-07-05T02:43:51.817286Z","shell.execute_reply.started":"2023-07-05T02:43:51.644344Z","shell.execute_reply":"2023-07-05T02:43:51.816244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualing Image Samples","metadata":{}},{"cell_type":"code","source":"disease_labels = {\n      0: \"Cassava Bacterial Blight (CBB)\",\n      1: \"Cassava Brown Streak Disease (CBSD)\",\n      2: \"Cassava Green Mottle (CGM)\",\n      3: \"Cassava Mosaic Disease (CMD)\",\n      4: \"Healthy\"\n    }\n\nplt.figure(figsize=(12, 12))\n\nfor i, (img, lab) in enumerate(train_ds.take(9)):\n    ax = plt.subplot(3,3,i+1)\n    plt.imshow(np.array(img[0]).astype(int))\n    plt.title(disease_labels[np.asarray(lab)[0]])\n    plt.axis(\"off\")\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:51.818547Z","iopub.execute_input":"2023-07-05T02:43:51.818885Z","iopub.status.idle":"2023-07-05T02:43:55.650034Z","shell.execute_reply.started":"2023-07-05T02:43:51.818856Z","shell.execute_reply":"2023-07-05T02:43:55.649010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing Images After Augmentation","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\n\nfor i, (img, lab) in enumerate(train_ds.take(1)):\n    print(img.shape)\n    \n    for i in range(9): # Plot first image\n        if i==1:\n            ax = plt.subplot(3,3,i+1)\n            plt.imshow(np.array(img[0]).astype(int))\n        else: # plot augmentated images\n            plt.axis('off')\n            ax = plt.subplot(3,3,i+1)\n            image, label = augmentation(img, lab)\n            plt.imshow(np.array(image[0]).astype(int))\n            plt.axis('off')\n","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:43:55.651293Z","iopub.execute_input":"2023-07-05T02:43:55.651724Z","iopub.status.idle":"2023-07-05T02:43:58.924941Z","shell.execute_reply.started":"2023-07-05T02:43:55.651692Z","shell.execute_reply":"2023-07-05T02:43:58.924043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Preparation - With Pretraied Resnet50","metadata":{}},{"cell_type":"code","source":"IMAGE_SHAPE = IMAGE_SIZE+(3,)\n\n\nwith strategy.scope():\n    \n    base_model = tf.keras.applications.ResNet50(input_shape=IMAGE_SHAPE,\n                                                include_top=False,\n                                                weights='imagenet'\n                                               )\n    \n    ## Making last few layers trainable and freezing the rest\n    print(\"Total Layers: \", len(base_model.layers))\n    freeze_layers_at = 100\n\n    for layer in base_model.layers[:freeze_layers_at]:\n        layer.trainable=False\n     \n    def make_model(input_shape):\n\n        inputs = tf.keras.Input(shape=input_shape)\n        x = base_model(inputs, training=False)\n        x = tfl.GlobalAveragePooling2D()(x)\n        x = tfl.Dense(64, activation='relu')(x)\n        x = tfl.Dropout(0.1)(x)\n        output = tfl.Dense(5, activation='softmax', kernel_regularizer=tf.keras.regularizers.L2(0.1))(x)\n\n        model = tf.keras.Model(inputs, output)\n\n        return model\n    \n    model = make_model(IMAGE_SHAPE)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:52:46.986470Z","iopub.execute_input":"2023-07-05T02:52:46.987491Z","iopub.status.idle":"2023-07-05T02:52:59.944352Z","shell.execute_reply.started":"2023-07-05T02:52:46.987449Z","shell.execute_reply":"2023-07-05T02:52:59.942888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n    loss = tf.keras.losses.SparseCategoricalCrossentropy(),\n    metrics=['sparse_categorical_accuracy']\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:52:59.946403Z","iopub.execute_input":"2023-07-05T02:52:59.946917Z","iopub.status.idle":"2023-07-05T02:53:00.011692Z","shell.execute_reply.started":"2023-07-05T02:52:59.946883Z","shell.execute_reply":"2023-07-05T02:53:00.010560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network Architecture","metadata":{}},{"cell_type":"code","source":"display(tf.keras.utils.plot_model(model))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:53:00.012870Z","iopub.execute_input":"2023-07-05T02:53:00.013141Z","iopub.status.idle":"2023-07-05T02:53:00.020141Z","shell.execute_reply.started":"2023-07-05T02:53:00.013115Z","shell.execute_reply":"2023-07-05T02:53:00.019185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define training epochs\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"cassava_base.h5\", \n                                                   save_best_only=True)\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=20,\n                                                     restore_best_weights=True\n                                                    )\nEPOCHS = 50\n\nSTEPS_PER_EPOCH = train_images_cnt // BATCH_SIZE\n\nhistory = model.fit(train_ds,\n                    validation_data=validation_ds,\n                    epochs=EPOCHS,\n                    steps_per_epoch=STEPS_PER_EPOCH,\n#                     class_weight=class_weights,\n                    callbacks=[checkpoint_cb, early_stopping_cb],\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:53:00.021951Z","iopub.execute_input":"2023-07-05T02:53:00.022569Z","iopub.status.idle":"2023-07-05T02:59:25.422820Z","shell.execute_reply.started":"2023-07-05T02:53:00.022541Z","shell.execute_reply":"2023-07-05T02:59:25.420927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"Loss Over Epochs\")\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:51:49.156711Z","iopub.execute_input":"2023-07-05T02:51:49.157054Z","iopub.status.idle":"2023-07-05T02:51:49.667518Z","shell.execute_reply.started":"2023-07-05T02:51:49.157026Z","shell.execute_reply":"2023-07-05T02:51:49.665812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image, label = next(iter(validation_ds))\nprediction = model.predict(image)\nprediction = np.argmax(prediction, axis=1)\nprint(prediction)\nprint(np.asarray(label))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T02:51:49.668797Z","iopub.execute_input":"2023-07-05T02:51:49.669143Z","iopub.status.idle":"2023-07-05T02:51:59.070548Z","shell.execute_reply.started":"2023-07-05T02:51:49.669094Z","shell.execute_reply":"2023-07-05T02:51:59.068970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}