{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Convert StratifiedKFold Images To TFRecord**","metadata":{}},{"cell_type":"markdown","source":"### **If you wonder what is TFRecord, check out link below!**\n\n* [Tensorflow.org](https://www.tensorflow.org/tutorials/load_data/tfrecord)","metadata":{}},{"cell_type":"markdown","source":"## **Reference**\n#### This notebook is based in **nice kernels** below!\n#### Appreciated for sharing kernels you all\n\n* [Kaveh Shahhosseini](https://www.kaggle.com/kavehshahhosseini/sartorius-convert-images-and-masks-to-tfrecord)\n* [Chris Deotte](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport os\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\n\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-14T13:23:37.840203Z","iopub.execute_input":"2021-11-14T13:23:37.840562Z","iopub.status.idle":"2021-11-14T13:23:43.116079Z","shell.execute_reply.started":"2021-11-14T13:23:37.840458Z","shell.execute_reply":"2021-11-14T13:23:43.115392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_WIDTH  = 704\nIMAGE_HEIGHT = 520\n\ndf = pd.read_csv(\"../input/sartorius-cell-instance-segmentation/train.csv\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-11-14T13:23:43.117242Z","iopub.execute_input":"2021-11-14T13:23:43.117923Z","iopub.status.idle":"2021-11-14T13:23:43.677432Z","shell.execute_reply.started":"2021-11-14T13:23:43.117891Z","shell.execute_reply":"2021-11-14T13:23:43.676760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Convert RLE Encoded Masks!**\n\n## What is RLE Encoding?\n\n#### It is a method of expressing only the number and repeated values that appear consecutively in the data.\n#### EXAMPLE : 11133333555 → 3 1 5 3 3 5\n\n* [Wikipedia](https://en.wikipedia.org/wiki/Run-length_encoding)\n* [dcode converter](https://www.dcode.fr/rle-compression)\n\n#### It will be easy to understand if you refer to the links above!","metadata":{}},{"cell_type":"code","source":"def rle_decode(mask_rle, shape):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.uint8)\n    for start, end in zip(starts, ends):\n        img[start : end] = 255\n    return img.reshape(shape)\n\n\ndef build_masks(labels, input_shape):\n    height, width = input_shape\n    mask = np.zeros((height, width, 1))\n    for label in labels:\n        mask += rle_decode(label, shape=(height, width, 1))\n    mask = mask.reshape((520, 704))\n    mask = mask.astype('uint8')\n    return mask","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:43.678641Z","iopub.execute_input":"2021-11-14T13:23:43.678859Z","iopub.status.idle":"2021-11-14T13:23:43.685917Z","shell.execute_reply.started":"2021-11-14T13:23:43.678834Z","shell.execute_reply":"2021-11-14T13:23:43.685132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **StratifiedKFold our ids of images!**","metadata":{}},{"cell_type":"markdown","source":"### **Check whether each id has one cell_type!**\n\n#### **If lengths of 'grouped' and 'ids' are same, there is only one cell_type in each id**","metadata":{}},{"cell_type":"code","source":"ids = df[\"id\"].unique()\ngrouped = df.groupby('id')['cell_type'].value_counts()\n\nif len(grouped) == len(ids):\n    print(\"Same!\")\nelse:\n    print('Nope!')","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:43.687665Z","iopub.execute_input":"2021-11-14T13:23:43.687909Z","iopub.status.idle":"2021-11-14T13:23:43.725347Z","shell.execute_reply.started":"2021-11-14T13:23:43.687881Z","shell.execute_reply":"2021-11-14T13:23:43.724527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Make a new list of folded ids!**","metadata":{}},{"cell_type":"code","source":"cell_data = grouped.index.to_frame().reset_index(drop = True)\nfor i in range(5):\n    cell_data[f'{i}_type'] = 'na'\ncell_data","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:43.726891Z","iopub.execute_input":"2021-11-14T13:23:43.727193Z","iopub.status.idle":"2021-11-14T13:23:43.753705Z","shell.execute_reply.started":"2021-11-14T13:23:43.727151Z","shell.execute_reply":"2021-11-14T13:23:43.752840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stkf = StratifiedKFold(n_splits = 5,\n                       shuffle = True,\n                       random_state = 2021)\n\nfor fold, (train_index, valid_index) in enumerate(stkf.split(cell_data['id'], cell_data['cell_type'])):\n    cell_data.loc[train_index, f'{fold}_type'] = 'train'\n    cell_data.loc[valid_index, f'{fold}_type'] = 'valid'\n\ncell_data","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:43.755109Z","iopub.execute_input":"2021-11-14T13:23:43.755567Z","iopub.status.idle":"2021-11-14T13:23:43.785285Z","shell.execute_reply.started":"2021-11-14T13:23:43.755526Z","shell.execute_reply":"2021-11-14T13:23:43.784531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 5, figsize = (12, 6))\n\nfor i in range(5):\n    sns.barplot(cell_data[cell_data[f'{i}_type'] == 'train']['cell_type'].value_counts().index,\n                cell_data[cell_data[f'{i}_type'] == 'train']['cell_type'].value_counts().values,\n                ax = ax[i]).set_title(f'train Fold {i}')\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:43.786325Z","iopub.execute_input":"2021-11-14T13:23:43.786546Z","iopub.status.idle":"2021-11-14T13:23:44.436680Z","shell.execute_reply.started":"2021-11-14T13:23:43.786513Z","shell.execute_reply":"2021-11-14T13:23:44.436139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 5, figsize = (12, 6))\n\nfor i in range(5):\n    sns.barplot(cell_data[cell_data[f'{i}_type'] == 'valid']['cell_type'].value_counts().index,\n                cell_data[cell_data[f'{i}_type'] == 'valid']['cell_type'].value_counts().values,\n                ax = ax[i]).set_title(f'valid Fold {i}')\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:44.437648Z","iopub.execute_input":"2021-11-14T13:23:44.438003Z","iopub.status.idle":"2021-11-14T13:23:45.220438Z","shell.execute_reply.started":"2021-11-14T13:23:44.437974Z","shell.execute_reply":"2021-11-14T13:23:45.219701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Seems Done!**","metadata":{}},{"cell_type":"markdown","source":"# **Convert Images to TFRecords!!**","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() \n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef serialize_example(image, label):\n    feature = {\n        'image': _bytes_feature(image.tobytes()),\n        'label': _bytes_feature(label.tobytes())\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:45.221679Z","iopub.execute_input":"2021-11-14T13:23:45.221977Z","iopub.status.idle":"2021-11-14T13:23:45.228217Z","shell.execute_reply.started":"2021-11-14T13:23:45.221937Z","shell.execute_reply":"2021-11-14T13:23:45.227410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir -p ./tfrecords/fold_0\n! mkdir -p ./tfrecords/fold_1\n! mkdir -p ./tfrecords/fold_2\n! mkdir -p ./tfrecords/fold_3\n! mkdir -p ./tfrecords/fold_4","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:23:45.231033Z","iopub.execute_input":"2021-11-14T13:23:45.231457Z","iopub.status.idle":"2021-11-14T13:23:49.223581Z","shell.execute_reply.started":"2021-11-14T13:23:45.231415Z","shell.execute_reply":"2021-11-14T13:23:49.222441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor fold in range(5):\n    print('Fold ', fold)\n    outpath = f\"./tfrecords/fold_{fold}\"\n    train_path = os.path.join(outpath,'train.tfrec')\n    valid_path = os.path.join(outpath,'valid.tfrec')\n    \n    train_ids = cell_data[cell_data[f'{fold}_type'] == \"train\"]['id'].values\n    valid_ids = cell_data[cell_data[f'{fold}_type'] == \"valid\"]['id'].values\n    \n    # Train Data TFRecord\n    with tf.io.TFRecordWriter(train_path, options=tf.io.TFRecordOptions(compression_type=\"GZIP\")) as writer:\n        for i in tqdm(train_ids, colour=\"#73d315\", ncols=100):\n            img_path = os.path.join(\"../input/sartorius-cell-instance-segmentation/train\", f\"{i}.png\")\n            img = cv2.imread(img_path)\n            img = cv2.resize(img, (256, 256))\n            img = (img/255.).astype('float32')\n            labels = df[df[\"id\"] == i][\"annotation\"].tolist()\n            mask = build_masks(labels, input_shape=(520, 704))\n            mask = cv2.resize(mask, (256, 256))\n            example = serialize_example(img, mask)\n            writer.write(example)\n    \n    # Valid Data TFRecord\n    with tf.io.TFRecordWriter(valid_path, options=tf.io.TFRecordOptions(compression_type=\"GZIP\")) as writer:\n        for i in tqdm(valid_ids, colour=\"#73d315\", ncols=100):\n            img_path = os.path.join(\"../input/sartorius-cell-instance-segmentation/train\", f\"{i}.png\")\n            img = cv2.imread(img_path)\n            img = (img/255.).astype('float32')\n            labels = df[df[\"id\"] == i][\"annotation\"].tolist()\n            mask = build_masks(labels, input_shape=(520, 704))\n            example = serialize_example(img, mask)\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T12:47:36.478885Z","iopub.execute_input":"2021-11-14T12:47:36.47915Z","iopub.status.idle":"2021-11-14T12:57:38.177351Z","shell.execute_reply.started":"2021-11-14T12:47:36.479116Z","shell.execute_reply":"2021-11-14T12:57:38.176217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Thanks!**","metadata":{}}]}