{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nfrom sklearn.model_selection import StratifiedKFold\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport shutil\nfrom functools import partial\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import class_weight\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nwith open('../input/cassava-leaf-disease-classification/label_num_to_disease_map.json') as file:\n    text = file.read()\nprint(text)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Original Distribution"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"figure = plt.figure(figsize=(8,4))\n(data['label'].value_counts()/len(data)*100).plot(kind='bar')\nplt.title(\"Distribution of Classes\")\nplt.ylabel('% count of classes')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Function for converting image to Tfrecord\nTaken from tensorflow site"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string), \n        'target': tf.io.FixedLenFeature([], tf.int64), \n        'image_name': tf.io.FixedLenFeature([], tf.string), \n    }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    target = example['target']\n    name = example['image_name']\n    return image, target, name\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        image, label, name = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(image[0])\n        plt.title(f\"{label[0]}: {name[0].numpy().decode('utf-8')}\", fontsize=12)\n    plt.show()\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, target, image_name):\n  feature = {\n      'image': _bytes_feature(image),\n      'target': _int64_feature(target),\n      'image_name': _bytes_feature(image_name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Image Downsampling"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a balanced data set\ndata['label'].value_counts()\nno_images_to_copy = 2580\nbalanced_df = data[data['label']==3].sample(frac=0.197)\nbalanced_df = pd.concat([balanced_df, \n                         data[data['label']==0],\n                        data[data['label']==1],\n                        data[data['label']==2],\n                        data[data['label']==4]],\n                       axis=0, ignore_index=True)\n\nbalanced_df = balanced_df.sample(frac=1).reset_index(drop=True)\ndisplay(balanced_df.head())\n\nfigure = plt.figure(figsize=(8,4))\n(balanced_df['label'].value_counts()/len(data)*100).plot(kind='bar')\nplt.title(\"Distribution of Classes\")\nplt.ylabel('% count of classes')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Writing an Image to TfRecord"},{"metadata":{"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nPATH = f'{database_base_path}train_images/'\nIMGS = balanced_df['image_id'].tolist()\nN_FILES = 10\nHEIGHT, WIDTH = (512, 512)\nIMG_QUALITY = 100\nprint('No of Images - ', len(IMGS))\ntrain = balanced_df.copy()\n\ndisplay(train.head())\n\n\n\nfolds = StratifiedKFold(n_splits=N_FILES, shuffle=True, random_state=123)\ntrain['file'] = -1\n\nfor fold_n, (train_idx, val_idx) in enumerate(folds.split(train, train['label'])):\n    print('File: %s has %s samples' % (fold_n+1, len(val_idx)))\n    train['file'].loc[val_idx] = fold_n\n    \ntrain.to_csv('train.csv', index=False)\n\n\n\n\n# Writing to TFRecords\nfor tfrec_num in range(N_FILES):\n    print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n    samples = train[train['file'] == tfrec_num]\n    n_samples = len(samples)\n    print(f'{n_samples} samples')\n    with tf.io.TFRecordWriter('Id_train%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n        for row in samples.itertuples():\n            label = row.label\n            image_name = row.image_id\n            img_path = f'{PATH}{image_name}'\n            \n            img = cv2.imread(img_path)\n            img = cv2.resize(img, (HEIGHT, WIDTH))\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n            \n            example = serialize_example(img, label, str.encode(image_name))\n            writer.write(example)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}