{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, math, glob, re\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-08T07:47:20.405063Z","iopub.execute_input":"2021-11-08T07:47:20.405761Z","iopub.status.idle":"2021-11-08T07:47:27.297100Z","shell.execute_reply.started":"2021-11-08T07:47:20.405655Z","shell.execute_reply":"2021-11-08T07:47:27.295997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"IMAGE_WIDTH  = 704\nIMAGE_HEIGHT = 520\n\ndf = pd.read_csv(\"../input/sartorius-cell-instance-segmentation/train.csv\")\nuuids = df[\"id\"].unique()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T07:47:27.298877Z","iopub.execute_input":"2021-11-08T07:47:27.299467Z","iopub.status.idle":"2021-11-08T07:47:28.037263Z","shell.execute_reply.started":"2021-11-08T07:47:27.299423Z","shell.execute_reply":"2021-11-08T07:47:28.036286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Images","metadata":{}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.uint8)\n    for start, end in zip(starts, ends):\n        img[start : end] = 1\n    return img.reshape(shape)\n\n\ndef build_masks(image_id, shape):\n    labels = df[df[\"id\"] == image_id][\"annotation\"].tolist()\n    masks = []\n    \n    for label in labels:\n        masks.append(rle_decode(label, shape=shape))\n            \n    return np.squeeze(np.stack(masks, axis=-1))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T07:53:21.712148Z","iopub.execute_input":"2021-11-08T07:53:21.712561Z","iopub.status.idle":"2021-11-08T07:53:21.722329Z","shell.execute_reply.started":"2021-11-08T07:53:21.712514Z","shell.execute_reply":"2021-11-08T07:53:21.721076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_filename = '0030fd0e6378'\nsample_path = os.path.join(\"../input/sartorius-cell-instance-segmentation/train\", f\"{sample_filename}.png\")\nsample_img = cv2.imread(sample_path)\nsample_img = (sample_img /255.).astype('float32')\nsample_masks = build_masks(sample_filename, shape=(IMAGE_HEIGHT, IMAGE_WIDTH, 1))\nprint(\"Image Shape: \", sample_img.shape, sample_img.dtype)\nprint(\"Label Shape: \", sample_masks.shape, sample_masks.dtype)\nfig, axs = plt.subplots(1, 2,figsize=(20, 20))\naxs[0].imshow(sample_img)\naxs[0].axis(\"off\")\naxs[1].imshow(np.sum(sample_masks, axis=-1))\naxs[1].axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T07:56:37.118889Z","iopub.execute_input":"2021-11-08T07:56:37.119353Z","iopub.status.idle":"2021-11-08T07:56:44.325601Z","shell.execute_reply.started":"2021-11-08T07:56:37.119312Z","shell.execute_reply":"2021-11-08T07:56:44.324310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert to TFRecord","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T07:59:58.432578Z","iopub.execute_input":"2021-11-08T07:59:58.432962Z","iopub.status.idle":"2021-11-08T07:59:58.441654Z","shell.execute_reply.started":"2021-11-08T07:59:58.432931Z","shell.execute_reply":"2021-11-08T07:59:58.440172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(image, label):\n    feature = {\n        'image': _bytes_feature(image.tobytes()),\n        'label': _bytes_feature(label.tobytes())\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T08:01:59.022748Z","iopub.execute_input":"2021-11-08T08:01:59.023480Z","iopub.status.idle":"2021-11-08T08:01:59.029344Z","shell.execute_reply.started":"2021-11-08T08:01:59.023420Z","shell.execute_reply":"2021-11-08T08:01:59.028408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n! mkdir -p ./tfrecords/\noutpath = \"./tfrecords\"\nwith tf.io.TFRecordWriter(os.path.join(outpath,'sartorius.tfrec'), options=tf.io.TFRecordOptions(compression_type=\"GZIP\")) as writer:\n    for i in tqdm(uuids, colour=\"#73d315\", ncols=100):\n        img_path = os.path.join(\"../input/sartorius-cell-instance-segmentation/train\", f\"{i}.png\")\n        img = cv2.imread(img_path)\n        img = (img/255.).astype('float32')\n        mask = build_masks(i, shape=(IMAGE_HEIGHT, IMAGE_WIDTH, 1))\n        example = serialize_example(img, mask)\n        writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T08:02:00.499577Z","iopub.execute_input":"2021-11-08T08:02:00.500272Z","iopub.status.idle":"2021-11-08T08:23:50.645177Z","shell.execute_reply.started":"2021-11-08T08:02:00.500212Z","shell.execute_reply":"2021-11-08T08:23:50.643969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Written Data ","metadata":{}},{"cell_type":"markdown","source":"## Deserialize TFRecord","metadata":{}},{"cell_type":"code","source":"def deserialize_example(serialized_string):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.string)\n    }\n    parsed_record = tf.io.parse_single_example(serialized_string, image_feature_description)\n    image = tf.reshape(tf.io.decode_raw(parsed_record['image'], tf.float32),(IMAGE_HEIGHT, IMAGE_WIDTH, 3))\n    label = tf.reshape(tf.io.decode_raw(parsed_record['label'], tf.uint8),(IMAGE_HEIGHT, IMAGE_WIDTH, -1))\n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-11-08T08:25:03.163085Z","iopub.execute_input":"2021-11-08T08:25:03.163850Z","iopub.status.idle":"2021-11-08T08:25:03.173751Z","shell.execute_reply.started":"2021-11-08T08:25:03.163799Z","shell.execute_reply":"2021-11-08T08:25:03.172238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = tf.data.TFRecordDataset(os.path.join(outpath,\"sartorius.tfrec\"), compression_type=\"GZIP\").map(deserialize_example)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T08:25:07.114940Z","iopub.execute_input":"2021-11-08T08:25:07.115703Z","iopub.status.idle":"2021-11-08T08:25:07.360359Z","shell.execute_reply.started":"2021-11-08T08:25:07.115663Z","shell.execute_reply":"2021-11-08T08:25:07.359224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot Data","metadata":{}},{"cell_type":"code","source":"ds = train_set.take(1)\nfor image, label in ds:\n    print(image.shape)\n    print(label.shape)\n\nfig, axs = plt.subplots(1, 2,figsize=(20, 20))\naxs[0].imshow(image)\naxs[0].axis('off')\naxs[1].imshow(np.sum(label, axis=-1))\naxs[1].axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T08:25:11.820753Z","iopub.execute_input":"2021-11-08T08:25:11.821225Z","iopub.status.idle":"2021-11-08T08:25:13.333448Z","shell.execute_reply.started":"2021-11-08T08:25:11.821181Z","shell.execute_reply":"2021-11-08T08:25:13.332451Z"},"trusted":true},"execution_count":null,"outputs":[]}]}