{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# IMPORTS","metadata":{}},{"cell_type":"code","source":"import re, math, os, cv2, random, warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-28T14:56:14.737071Z","iopub.execute_input":"2024-04-28T14:56:14.737518Z","iopub.status.idle":"2024-04-28T14:56:29.287575Z","shell.execute_reply.started":"2024-04-28T14:56:14.737490Z","shell.execute_reply":"2024-04-28T14:56:29.286768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string), \n        'target': tf.io.FixedLenFeature([], tf.int64), \n        'image_name': tf.io.FixedLenFeature([], tf.string), \n    }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    target = example['target']\n    name = example['image_name']\n    return image, target, name\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        image, label, name = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(image[0])\n        plt.title(f\"{label[0]}: {name[0].numpy().decode('utf-8')}\", fontsize=12)\n    plt.show()\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, target, image_name):\n  feature = {\n      'image': _bytes_feature(image),\n      'target': _int64_feature(target),\n      'image_name': _bytes_feature(image_name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.289307Z","iopub.execute_input":"2024-04-28T14:56:29.289877Z","iopub.status.idle":"2024-04-28T14:56:29.306077Z","shell.execute_reply.started":"2024-04-28T14:56:29.289849Z","shell.execute_reply":"2024-04-28T14:56:29.305036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PARAMETER DECLARATION","metadata":{}},{"cell_type":"markdown","source":"You can modify the Height and Width of the Image based on your model. Also you can modify the number of File Splits you require as well.","metadata":{}},{"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nPATH = f'{database_base_path}train_images/'\nIMGS = os.listdir(PATH)\nN_FILES = 5 # split images into 10 files\nHEIGHT, WIDTH = (512, 512)\nIMG_QUALITY = 100\n\nprint(f'Image samples: {len(IMGS)}')","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.307305Z","iopub.execute_input":"2024-04-28T14:56:29.307661Z","iopub.status.idle":"2024-04-28T14:56:29.683906Z","shell.execute_reply.started":"2024-04-28T14:56:29.307627Z","shell.execute_reply":"2024-04-28T14:56:29.682918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(database_base_path + 'train.csv')\nprint('Train samples: %d' % len(train))\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.686558Z","iopub.execute_input":"2024-04-28T14:56:29.687016Z","iopub.status.idle":"2024-04-28T14:56:29.723196Z","shell.execute_reply.started":"2024-04-28T14:56:29.686982Z","shell.execute_reply":"2024-04-28T14:56:29.722261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for NaN values in the DataFrame\nnan_counts = train.isna().sum()\n\n# Print the count of NaN values in each column\nprint(\"Count of NaN values per column:\")\nprint(nan_counts)","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.724420Z","iopub.execute_input":"2024-04-28T14:56:29.724818Z","iopub.status.idle":"2024-04-28T14:56:29.735520Z","shell.execute_reply.started":"2024-04-28T14:56:29.724782Z","shell.execute_reply":"2024-04-28T14:56:29.734517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate Files","metadata":{}},{"cell_type":"code","source":"folds = StratifiedKFold(n_splits=N_FILES, shuffle=True, random_state=seed)\ntrain['file'] = -1\n\nfor fold_n, (train_idx, val_idx) in enumerate(folds.split(train, train['label'])):\n    print('File: %s has %s samples' % (fold_n+1, len(val_idx)))\n    train['file'].loc[val_idx] = fold_n\n    \ndisplay(train.head())\ndisplay(train.describe())\ntrain.to_csv('train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.736762Z","iopub.execute_input":"2024-04-28T14:56:29.737075Z","iopub.status.idle":"2024-04-28T14:56:29.818061Z","shell.execute_reply.started":"2024-04-28T14:56:29.737051Z","shell.execute_reply":"2024-04-28T14:56:29.817081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate TF Records","metadata":{}},{"cell_type":"code","source":"for tfrec_num in range(N_FILES):\n    print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n    samples = train[train['file'] == tfrec_num]\n    n_samples = len(samples)\n    print(f'{n_samples} samples')\n    with tf.io.TFRecordWriter('Id_train%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n        for row in samples.itertuples():\n            label = row.label\n            image_name = row.image_id\n            img_path = f'{PATH}{image_name}'\n            \n            img = cv2.imread(img_path)\n            img = cv2.resize(img, (HEIGHT, WIDTH))\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n            \n            example = serialize_example(img, label, str.encode(image_name))\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2024-04-28T14:56:29.819262Z","iopub.execute_input":"2024-04-28T14:56:29.819561Z","iopub.status.idle":"2024-04-28T15:05:14.438609Z","shell.execute_reply.started":"2024-04-28T14:56:29.819536Z","shell.execute_reply":"2024-04-28T15:05:14.437738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nFILENAMES = tf.io.gfile.glob('Id_train*.tfrec')\nprint(f'TFRecords files: {FILENAMES}')\nprint(f'Created image samples: {count_data_items(FILENAMES)}')\n\ndisplay_samples(load_dataset(FILENAMES, HEIGHT, WIDTH).batch(1), 6, 6)","metadata":{"execution":{"iopub.status.busy":"2024-04-28T15:05:14.439779Z","iopub.execute_input":"2024-04-28T15:05:14.440064Z","iopub.status.idle":"2024-04-28T15:05:18.652646Z","shell.execute_reply.started":"2024-04-28T15:05:14.440040Z","shell.execute_reply":"2024-04-28T15:05:18.651198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Complete Set Label Distribution\n","metadata":{}},{"cell_type":"code","source":"CLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\n\nlabel_count = train.groupby('label', as_index=False).count()\nlabel_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\nlabel_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\nfig, ax = plt.subplots(1, 1, figsize=(14, 8))\nfig.suptitle(f'Barchart Showing the Image Distribution amongst Classes in the Dataset', fontsize=22)\nax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-28T15:13:34.807086Z","iopub.execute_input":"2024-04-28T15:13:34.808046Z","iopub.status.idle":"2024-04-28T15:13:35.044108Z","shell.execute_reply.started":"2024-04-28T15:13:34.808011Z","shell.execute_reply":"2024-04-28T15:13:35.043096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Distribution for each File","metadata":{}},{"cell_type":"code","source":"for fold_n in range(folds.n_splits):\n    label_count = train[train['file'] == fold_n].groupby('label', as_index=False).count()\n    label_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\n    label_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\n    fig, ax = plt.subplots(1, 1, figsize=(14, 8))\n    fig.suptitle(f'File {fold_n+1}', fontsize=22)\n    ax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\n    ax.tick_params(labelsize=16)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-28T15:05:18.959026Z","iopub.execute_input":"2024-04-28T15:05:18.959327Z","iopub.status.idle":"2024-04-28T15:05:20.465045Z","shell.execute_reply.started":"2024-04-28T15:05:18.959300Z","shell.execute_reply":"2024-04-28T15:05:20.464155Z"},"trusted":true},"execution_count":null,"outputs":[]}]}