{"cells":[{"metadata":{},"cell_type":"markdown","source":"# How To Create TFRecords\n"},{"metadata":{},"cell_type":"markdown","source":"# Load Meta Data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# LOAD LIBRARIES\nimport numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PATHS TO IMAGES\nPATH = '../input/oc-d-train-df/train_images_128/train_images_128/'\n#PATH2 = '../input/jpeg-melanoma-256x256/test/'\nIMGS = os.listdir(PATH); \n#IMGS2 = os.listdir(PATH2)\nprint('There are %i train '%(len(IMGS)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOAD TRAIN META DATA\ntrain_df = pd.read_csv('../input/oc-csv-file/ocular.csv')\n#df.rename({'image_id':'image_name'},axis=1,inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOAD TEST META DATA\n#test = pd.read_csv('../input/jpeg-melanoma-256x256/test.csv')\n#test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## My code here to check data properties"},{"metadata":{"trusted":true},"cell_type":"code","source":"#df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df[df['age_approx'].isnull()].head()\n#df[df['anatom_site_general_challenge'].isnull()].head()\n#df[df['patient_id'].isnull()].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df.shape   # (60487, 7)\n#test.shape  # (10982, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df.columns    ## Index(['patient_id', 'image_name', 'target', 'source', 'sex', 'age_approx','anatom_site_general_challenge'],dtype='object')\n# test.columns  ## Index(['image_name', 'patient_id', 'sex', 'age_approx','anatom_site_general_challenge'],dtype='object')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Label Encode Meta Data\nIt is more efficient to store this meta data as integers instead of strings. We will impute the Age NaNs to Age mean. Then all other NaNs will be convert to `-1` and the other strings will be converted to `0, 1, 2, 3, ...` in the order they appear in the printed lists below."},{"metadata":{"trusted":true},"cell_type":"code","source":"# COMBINE TRAIN AND TEST TO ENCODE TOGETHER\n#cols = test.columns\n#comb = pd.concat([df[cols],test[cols]],ignore_index=True,axis=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#comb.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# comb.shape  ## (71469, 5)\n#comb['patient_id'].factorize()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LABEL ENCODE ALL STRINGS\ncats = ['Patient Sex','filepath','labels','target','filename']\nfor c in cats:\n    comb[c],mp = comb[c].factorize()\n    print(mp)\nprint('Imputing Age NaN count =',comb.age_approx.isnull().sum())\ncomb.age_approx.fillna(comb.age_approx.mean(),inplace=True)\ncomb['Patient Age'] = comb.age_approx.astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# REWRITE DATA TO DATAFRAMES\ndf[cols] = comb.loc[:df.shape[0]-1,cols].values\ntest[cols] = comb.loc[df.shape[0]:,cols].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#LABEL ENCODE TRAIN SOURCE\ndf.source,mp = df.source.factorize()\nprint(mp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Write TFRecords - Train\nAll the code below comes from TensorFlow's docs [here][1]\n\n[1]: https://www.tensorflow.org/tutorials/load_data/tfrecord"},{"metadata":{"trusted":true},"cell_type":"code","source":"type(tf.constant(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2, feature3, feature4, feature5, feature6, feature7):\n  feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'patient_id': _int64_feature(feature2),\n      'sex': _int64_feature(feature3),\n      'age_approx': _int64_feature(feature4),\n      'anatom_site_general_challenge': _int64_feature(feature5),\n      'source': _int64_feature(feature6),\n      'target': _int64_feature(feature7)\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 2071\nCT = len(IMGS)//SIZE + int(len(IMGS)%SIZE!=0)\nfor j in range(CT):\n    print(); print('Writing TFRecord %i of %i...'%(j,CT))\n    CT2 = min(SIZE,len(IMGS)-j*SIZE)\n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(j,CT2)) as writer:\n        for k in range(CT2):\n            img = cv2.imread(PATH+IMGS[SIZE*j+k])\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n            name = IMGS[SIZE*j+k].split('.')[0]\n            row = df.loc[df.image_name==name]\n            example = serialize_example(\n                img, str.encode(name),\n                row.patient_id.values[0],\n                row.sex.values[0],\n                row.age_approx.values[0],                        \n                row.anatom_site_general_challenge.values[0],\n                row.source.values[0],\n                row.target.values[0])\n            writer.write(example)\n            if k%100==0: print(k,', ',end='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls -l","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Write TFRecords - Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"def serialize_example2(feature0, feature1, feature2, feature3, feature4, feature5): \n  feature = {\n      'image': _bytes_feature(feature0),\n      'image_name': _bytes_feature(feature1),\n      'patient_id': _int64_feature(feature2),\n      'sex': _int64_feature(feature3),\n      'age_approx': _int64_feature(feature4),\n      'anatom_site_general_challenge': _int64_feature(feature5),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 687\nCT = len(IMGS2)//SIZE + int(len(IMGS2)%SIZE!=0)\nfor j in range(CT):\n    print(); print('Writing TFRecord %i of %i...'%(j,CT))\n    CT2 = min(SIZE,len(IMGS2)-j*SIZE)\n    with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(j,CT2)) as writer:\n        for k in range(CT2):\n            img = cv2.imread(PATH2+IMGS2[SIZE*j+k])\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n            name = IMGS2[SIZE*j+k].split('.')[0]\n            row = test.loc[test.image_name==name]\n            example = serialize_example2(\n                img, str.encode(name),\n                row.patient_id.values[0],\n                row.sex.values[0],\n                row.age_approx.values[0],                        \n                row.anatom_site_general_challenge.values[0])\n            writer.write(example)\n            if k%100==0: print(k,', ',end='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Verify TFRecords\nWe will verify the TFRecords we just made by using code from the Flower Comp starter notebook [here][1] to display the TFRecords below.\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\nCLASSES = [0,1]\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    #if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n    #    numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = label\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = example['image_name']\n    return image, label # returns a dataset of (image, label) pairs\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# INITIALIZE VARIABLES\nIMAGE_SIZE= [512,512]; BATCH_SIZE = 32\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob('train*.tfrec')\nprint('There are %i train images'%count_data_items(TRAINING_FILENAMES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DISPLAY TRAIN IMAGES\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = iter(training_dataset)\n\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}