{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This notebook demonstrate use of TF Reocords because TPU training is optimized with TF Record input pipe line"},{"metadata":{},"cell_type":"markdown","source":"# Content\n1. Setup\n2. Data processing\n3. Visualize sample batch\n4. Build and train model\n5. Make predictions\n6. Tips for improvement\n"},{"metadata":{},"cell_type":"markdown","source":"# Setup"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nfrom kaggle_datasets import KaggleDatasets\nimport json\nfrom functools import partial\nprint(\"TensorFlow version: \", tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"Device:\", tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('#Replicas: ', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n0, n1, n2, n3, n4 = np.bincount(train_df['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total = train_df.shape[0]\ntotal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\nIMAGE_SIZE = [512, 512]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FILENAMES = tf.io.gfile.glob(GCS_PATH + \"/train_tfrecords/*.tfrec\")\ntrain_split = int(0.7 * len(FILENAMES))\nval_split = int(0.2 * len(FILENAMES))\nTRAINING_FILENAMES, VAL_FILENAMES, EVAL_FILENAMES = FILENAMES[:train_split], FILENAMES[train_split:train_split+val_split],FILENAMES[train_split+val_split:]\n\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + \"/test_tfrecords/*.tfrec\")\nprint(\"Total labeled files: \",len(FILENAMES))\nprint(\"Train TFRecord Files:\", len(TRAINING_FILENAMES))\nprint(\"Validation TFRecord Files:\", len(VAL_FILENAMES))\nprint(\"Evalution TFRecord Files:\", len(EVAL_FILENAMES))\nprint(\"Test TFRecord Files:\", len(TEST_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get order of IDs of test data, so that we can create submission file accordingly"},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_FILENAMES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_raw_dataset = tf.data.TFRecordDataset(TEST_FILENAMES)\ntest_image_feature_description = {\n\n    \"image\": tf.io.FixedLenFeature([],tf.string),\n    \"image_name\": tf.io.FixedLenFeature([],tf.string)\n}\n\n#read below comment to know about feature description\n\ndef _parse_image_function_test(example_proto):\n  # Parse the input tf.train.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, test_image_feature_description)\n\ntest_parsed_image_dataset = test_raw_dataset.map(_parse_image_function_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To know, how one can get/ know feature description, have a loot at this [notebook](https://www.kaggle.com/senkmp/simple-eda-using-tf-records) https://www.kaggle.com/senkmp/simple-eda-using-tf-records"},{"metadata":{"trusted":true},"cell_type":"code","source":"ids = []\nfor image_features in test_parsed_image_dataset:\n  id = image_features['image_name'].numpy().decode(\"utf-8\") \n  ids.append(id)\nlen(ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Input Pipe line"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_tfrecord(example, labeled):\n    tfrecord_format = (\n        {\n        \"image\": tf.io.FixedLenFeature([],tf.string),\n        \"image_name\": tf.io.FixedLenFeature([],tf.string),\n        \"target\": tf.io.FixedLenFeature([],tf.int64)\n        }\n        if labeled\n        else { \"image\": tf.io.FixedLenFeature([],tf.string),\n                \"image_name\": tf.io.FixedLenFeature([],tf.string),}\n    )\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example[\"image\"])\n    if labeled:\n        label = [tf.cast(example[\"target\"], tf.int64)]\n        return image, label\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, labeled=True,order = True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = not order  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames)  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order)  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n    return dataset\n\ndef get_dataset(filenames, labeled=True,shuffle=True,order=True):\n    \n    dataset = load_dataset(filenames, labeled=labeled,order=order)\n    if shuffle:\n        dataset = dataset.shuffle(2048)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES)\nval_dataset = get_dataset(VAL_FILENAMES)\neval_dataset = get_dataset(EVAL_FILENAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = get_dataset(TEST_FILENAMES, labeled=False,shuffle=False,order=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize sample batch"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_file = '../input/cassava-leaf-disease-classification/label_num_to_disease_map.json'\nf= open(labels_file)\nlabels = json.load(f)\nf.close()\nlabels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_batch, label_batch = next(iter(train_dataset))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_batch(image_batch, label_batch):\n    plt.figure(figsize=(15, 15))\n    for n in range(25):\n        ax = plt.subplot(5, 5, n + 1)\n        plt.imshow(image_batch[n] / 255.0)\n        t = label_batch[n]\n        \n        title = labels[str(t[0])]\n        plt.title(title)\n        plt.axis(\"off\")\n    plt.tight_layout()\n\n\n\nshow_batch(image_batch.numpy(), label_batch.numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build and train model"},{"metadata":{},"cell_type":"markdown","source":"## Give them equal rights :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_for_0 = (1/n0)*total/5\nweight_for_1 = (1/n1)*total/5\nweight_for_2 = (1/n2)*total/5\nweight_for_3 = (1/n3)*total/5\nweight_for_4 = (1/n4)*total/5\nprint(weight_for_0,weight_for_1,weight_for_2,weight_for_3,weight_for_4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weight = {0: weight_for_0, 1: weight_for_1, 2: weight_for_2, 3: weight_for_3, 4: weight_for_4}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncheckpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', save_best_only=True, monitor='val_loss', mode='min')\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(\n    patience=10, restore_best_weights=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  def make_model1():\n    base_model = tf.keras.applications.MobileNet(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=\"imagenet\"\n    )\n\n    base_model.trainable = False\n\n    inputs = tf.keras.layers.Input([*IMAGE_SIZE, 3])\n    \n    x = base_model(inputs, training=False)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n   \n    outputs = tf.keras.layers.Dense(5, activation=\"softmax\")(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model.compile(\n        optimizer='adam',\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy'])\n    # used sparse_categorical_crossentropy instead of categorical_crossentropy because output is not one hot vector\n    # categorical_accuracy == accuracy for this data\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''import efficientnet.efficientnet.tfkeras as efn \n  def make_model():\n    base_model = efn.EfficientNetB7(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False, weights=\"imagenet\"\n    )\n\n    base_model.trainable = False\n\n    inputs = tf.keras.layers.Input([*IMAGE_SIZE, 3])\n    \n    x = base_model(inputs, training=False)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n   \n    outputs = tf.keras.layers.Dense(5, activation=\"softmax\")(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model.compile(\n        optimizer='adam',\n        loss='sparse_categorical_crossentropy',\n        metrics=['accuracy'])\n    # used sparse_categorical_crossentropy instead of categorical_crossentropy because output is not one hot vector\n    # categorical_accuracy == accuracy for this data\n\n    return model'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = make_model1()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    train_dataset,\n    epochs=100,\n    validation_data=val_dataset,\n    callbacks=[early_stopping_cb, checkpoint],\n    class_weight=class_weight\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate(eval_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(test_dataset)\npred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_classes  = pred.argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame()\nsubmission_df['image_id'] = ids\nsubmission_df['label'] = pred_classes\nsubmission_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tips for improvement\n1. Thing about imbalanced classes\n2. Data augmentation\n3. Better model and fine tuning\n"},{"metadata":{},"cell_type":"markdown","source":"**Let me know in comment section if this is helpful ⬆️**"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}