{"cells":[{"metadata":{"papermill":{"duration":0.005277,"end_time":"2020-09-01T23:49:44.630619","exception":false,"start_time":"2020-09-01T23:49:44.625342","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dependencies\n\n- Discussion [thread](https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/198744)\n- Reference: [How To Create TFRecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-09-01T23:49:44.648219Z","iopub.status.busy":"2020-09-01T23:49:44.646993Z","iopub.status.idle":"2020-09-01T23:49:53.618519Z","shell.execute_reply":"2020-09-01T23:49:53.617575Z"},"papermill":{"duration":8.982954,"end_time":"2020-09-01T23:49:53.618672","exception":false,"start_time":"2020-09-01T23:49:44.635718","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import re, math, os, cv2, random, warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"os.mkdir('/kaggle/working/train/')\n!cp -r /kaggle/input/cassava-leaf-disease-classification/train_images/* train # move data to \"train\" folder","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Find duplicates"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import glob, torch, imagehash\nfrom tqdm.auto import tqdm\nfrom PIL import Image\n\n\nIMAGES_DIR = '/kaggle/working/train/'\n\nfuncs = [\n        imagehash.average_hash,\n        imagehash.phash,\n        imagehash.dhash,\n        imagehash.whash,\n    ]\nimage_ids = []\nhashes = []\n\nfor path in tqdm(glob.glob(IMAGES_DIR + '*.jpg')):\n    image = Image.open(path)\n    image_id = os.path.basename(path)\n    image_ids.append(image_id)\n    hashes.append(np.array([f(image).hash for f in funcs]).reshape(256))\n\nhashes_all = np.array(hashes)\nhashes_all = torch.Tensor(hashes_all.astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sims = np.array([(hashes_all[i] == hashes_all).sum(dim=1).numpy()/256 for i in range(hashes_all.shape[0])])\n\nindices1 = np.where(sims > 0.9)\nindices2 = np.where(indices1[0] != indices1[1])\nimage_ids1 = [image_ids[i] for i in indices1[0][indices2]]\nimage_ids2 = [image_ids[i] for i in indices1[1][indices2]]\ndups = {tuple(sorted([image_id1,image_id2])):True for image_id1, image_id2 in zip(image_ids1, image_ids2)}\nduplicate_image_ids = sorted(list(dups))\nprint('found %d duplicates' % len(duplicate_image_ids))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Remove duplicates from external data\nimgs_to_remove = [x[1] for x in duplicate_image_ids]\nremove_pd = []\nfor image in imgs_to_remove:\n    remove_pd.append(image)        ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"!rm -r /kaggle/working/train/ # delete image files","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.005522,"end_time":"2020-09-01T23:49:53.63003","exception":false,"start_time":"2020-09-01T23:49:53.624508","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Auxiliar functions"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-01T23:49:53.664561Z","iopub.status.busy":"2020-09-01T23:49:53.659015Z","iopub.status.idle":"2020-09-01T23:49:53.674019Z","shell.execute_reply":"2020-09-01T23:49:53.673148Z"},"papermill":{"duration":0.038443,"end_time":"2020-09-01T23:49:53.674156","exception":false,"start_time":"2020-09-01T23:49:53.635713","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string), \n        'target': tf.io.FixedLenFeature([], tf.int64), \n        'image_name': tf.io.FixedLenFeature([], tf.string), \n    }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    target = example['target']\n    name = example['image_name']\n    return image, target, name\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        image, label, name = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(image[0])\n        plt.title(f\"{label[0]}: {name[0].numpy().decode('utf-8')}\", fontsize=12)\n    plt.show()\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, target, image_name):\n  feature = {\n      'image': _bytes_feature(image),\n      'target': _int64_feature(target),\n      'image_name': _bytes_feature(image_name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.004967,"end_time":"2020-09-01T23:49:53.684592","exception":false,"start_time":"2020-09-01T23:49:53.679625","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Parameters"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-09-01T23:49:53.702127Z","iopub.status.busy":"2020-09-01T23:49:53.701251Z","iopub.status.idle":"2020-09-01T23:49:53.737166Z","shell.execute_reply":"2020-09-01T23:49:53.736353Z"},"papermill":{"duration":0.047388,"end_time":"2020-09-01T23:49:53.737327","exception":false,"start_time":"2020-09-01T23:49:53.689939","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"database_base_path = '/kaggle/input/cassava-leaf-disease-classification/'\nPATH = f'{database_base_path}train_images/'\nIMGS = os.listdir(PATH)\nN_FILES = 15 # split images into 15 files\nHEIGHT, WIDTH = (256, 256)\nIMG_QUALITY = 100\n\nprint(f'Image samples: {len(IMGS)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train = pd.read_csv(database_base_path + 'train.csv')\n\n# Remove duplicates from train data\ntrain = train[~train['image_id'].isin(remove_pd)]\ntrain.reset_index(inplace=True)\nprint('Train samples: %d' % len(train))\n\ndisplay(train.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split samples into 15 different files"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=N_FILES, shuffle=True, random_state=seed)\ntrain['file'] = -1\n\nfor fold_n, (train_idx, val_idx) in enumerate(folds.split(train, train['label'])):\n    print('File: %s has %s samples' % (fold_n+1, len(val_idx)))\n    train['file'].loc[val_idx] = fold_n\n    \ndisplay(train.head())\ndisplay(train.describe())\ntrain.to_csv('train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.005311,"end_time":"2020-09-01T23:49:53.747958","exception":false,"start_time":"2020-09-01T23:49:53.742647","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Generate TF records"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"for tfrec_num in range(N_FILES):\n    print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n    samples = train[train['file'] == tfrec_num]\n    n_samples = len(samples)\n    print(f'{n_samples} samples')\n    with tf.io.TFRecordWriter('Id_train%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n        for row in samples.itertuples():\n            label = row.label\n            image_name = row.image_id\n            img_path = f'{PATH}{image_name}'\n            \n            img = cv2.imread(img_path)\n            img = cv2.resize(img, (HEIGHT, WIDTH))\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n            \n            example = serialize_example(img, label, str.encode(image_name))\n            writer.write(example)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.00591,"end_time":"2020-09-01T23:50:01.60518","exception":false,"start_time":"2020-09-01T23:50:01.59927","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Visualize created TF records\n\n## Class map\n\n```\n0: Cassava Bacterial Blight (CBB)\n1: Cassava Brown Streak Disease (CBSD)\n2: Cassava Green Mottle (CGM)\n3: Cassava Mosaic Disease (CMD)\n4: Healthy\n```"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-01T23:50:01.659353Z","iopub.status.busy":"2020-09-01T23:50:01.658489Z","iopub.status.idle":"2020-09-01T23:50:04.708827Z","shell.execute_reply":"2020-09-01T23:50:04.709585Z"},"papermill":{"duration":3.070142,"end_time":"2020-09-01T23:50:04.709782","exception":false,"start_time":"2020-09-01T23:50:01.63964","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nFILENAMES = tf.io.gfile.glob('Id_train*.tfrec')\nprint(f'TFRecords files: {FILENAMES}')\nprint(f'Created image samples: {count_data_items(FILENAMES)}')\n\ndisplay_samples(load_dataset(FILENAMES, HEIGHT, WIDTH).batch(1), 6, 6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Complete set label distribution"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"CLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\n\nlabel_count = train.groupby('label', as_index=False).count()\nlabel_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\nlabel_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\nfig, ax = plt.subplots(1, 1, figsize=(14, 8))\nax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Labels distribution for each file"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"for fold_n in range(folds.n_splits):\n    label_count = train[train['file'] == fold_n].groupby('label', as_index=False).count()\n    label_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\n    label_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\n    fig, ax = plt.subplots(1, 1, figsize=(14, 8))\n    fig.suptitle(f'File {fold_n+1}', fontsize=22)\n    ax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\n    ax.tick_params(labelsize=16)\n\n    plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}