{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About This Notebook\n\nThis is the general process for writing tfrecords for the ALASKA2 competition. \n\nPlease note that kaggle kernels can only store 5gb, so you need to fork it many times to split the data into datsets of 5gb. \n\nIf you use the public option, you won't be consuming your private data storage space. \n\nThe dataframe preparation is taken from https://www.kaggle.com/hooong/train-inference-gpu-baseline. \n\nI recommend reading https://www.tensorflow.org/tutorials/load_data/tfrecord#writing_a_tfrecord_file for more information if you plan on doing fancy stuff. \n\nJust remember that the augmentations that you can do on the TPU are limited. Most augmentations libraries do not work on the TPU (albumentations for example). You may have to pre augment your data. \n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport tensorflow as tf ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(bits, label):\n    feature = {\n        'bits': _bytes_feature(bits),\n        'label' : _int64_feature(label)\n      }\n\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n\ndef tf_serialize_example(bits, label):\n    tf_string = tf.py_function(\n        serialize_example,\n        (bits, label),  # pass these args to the above function.\n        tf.string)      # the return type is `tf.string`.\n    return tf.reshape(tf_string, ()) \n\ndef read_image(filename, label=None):\n    bits = tf.io.read_file(filename)\n    if label is None:\n        return bits\n    else:\n        return bits, label\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob \nimport random\nfrom sklearn.model_selection import GroupKFold\nimport pandas as pd \n\ndataset = []\n\nfor label, kind in enumerate(['Cover', 'JMiPOD', 'JUNIWARD', 'UERD']):\n    for path in glob('../input/alaska2-image-steganalysis/Cover/*.jpg'):\n        dataset.append({\n            'kind': kind,\n            'image_name': path.split('/')[-1],\n            'label': label\n        })\n\nrandom.shuffle(dataset)\ndataset = pd.DataFrame(dataset)\n\ngkf = GroupKFold(n_splits=5)\n\ndataset.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(gkf.split(X=dataset.index, y=dataset['label'], groups=dataset['image_name'])):\n    dataset.loc[dataset.iloc[val_index].index, 'fold'] = fold_number\n\n\ndataset['local_path'] = dataset.apply(lambda x : '../input/alaska2-image-steganalysis/%s/%s' % (x.kind, x.image_name), axis = 1)\ndataset = dataset.sort_values('image_name')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm \n\nstart = 0\n\n# 30 records at ~150mb each \nfor i in tqdm(range(start * 30, start *30 + 30)):\n    \n    df = dataset.iloc[1500 * i : 1500 * (i + 1)]\n\n    ds = tf.data.Dataset.from_tensor_slices((df.local_path.values, df.label.values))\n\n    ds = ds.map(read_image)\n    ds = ds.map(tf_serialize_example)\n\n    def generator():\n        for features in ds:\n            yield features\n\n    serialized_ds = tf.data.Dataset.from_generator(generator, output_types=tf.string, output_shapes=())\n\n\n    serialized_ds\n\n    filename = '%05d.tfrecord' % i\n    writer = tf.data.experimental.TFRecordWriter(filename)\n    writer.write(serialized_ds)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_dataset = tf.data.TFRecordDataset(filename)\n\n# Create a description of the features.\nfeature_description = {\n    'bits': tf.io.FixedLenFeature([], tf.string, default_value=''),\n    'label' : tf.io.FixedLenFeature([], tf.int64, default_value=0),\n}\n\ndef _parse_function(example_proto):\n  # Parse the input `tf.Example` proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, feature_description)\n\ndef split(features):\n    return features['bits'], features['label']\n\ndef decode_image(bits, label=None, image_size=(512, 512)):\n    \n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\np = raw_dataset.map(_parse_function)\np = p.map(split)\np = p.map(decode_image)\n\nfor a in p.take(1):\n    print(a)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}