{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport argparse\nimport os\n\n\ndef normalize_image(img, org_width, org_height, new_width, new_height):\n  # Invert\n  img = 255 - img\n  # Normalize\n  img = (img * (255.0 / img.max())).astype(np.uint8)\n  # Reshape\n  img = img.reshape(org_height, org_width)\n  image_resized = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_LINEAR)\n  return image_resized\n\n\ndef dump_images(args, org_width, org_height, new_width, new_height):\n  labels = pd.read_csv(args.labels)\n  iids = labels['image_id']\n  root = labels['grapheme_root']\n  vowel = labels['vowel_diacritic']\n  consonant = labels['consonant_diacritic']\n  labels = {a: (b, c, d) for a, b, c, d in zip(iids, root, vowel, consonant)}\n  tuples = sorted(set(labels.values()))\n  tuples_to_int = {v: k for k, v in enumerate(tuples)}\n  print(f'Got {len(tuples)} unique combinations')\n  for i in tqdm(range(0, 4)):\n    df = pd.read_parquet(args.data_template % i)\n    image_ids = df['image_id'].values\n    df = df.drop(['image_id'], axis=1)\n    for image_id, index in tqdm(zip(image_ids, range(df.shape[0])), total=df.shape[0]):\n      normalized = normalize_image(df.loc[df.index[index]].values,\n          org_width, org_height, new_width, new_height)\n      r, v, c = labels[image_id]\n      tuple_int = tuples_to_int[(r, v, c)]\n      # e.g: 'Train_300_rt_29_vl_5_ct_0_ti_179.png'\n      out_fn = os.path.join(args.image_dir, f'{image_id}_rt_{r}_vl_{v}_ct_{c}_ti_{tuple_int}.png')\n      cv2.imwrite(out_fn, normalized)\n\n\ndef main():\n  parser = argparse.ArgumentParser()\n  parser.add_argument('--image_dir', type=str, default='images')\n  parser.add_argument('--data_template', type=str, default='../input/bengaliai-cv19/train_image_data_%d.parquet')\n  parser.add_argument('--labels', type=str, default='../input/bengaliai-cv19/train.csv')\n  args, _ = parser.parse_known_args()\n\n  os.makedirs(args.image_dir, exist_ok=True)\n\n  org_height = 137\n  org_width = 236\n  new_height = 160  # 5 * 32\n  new_width = 256  # 8 * 32\n  dump_images(args, org_width, org_height, new_width, new_height)\n  print(f'Done wrote to {args.image_dir}')\n\nmain()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Copyright 2020 Google LLC\n\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n\n# http://www.apache.org/licenses/LICENSE-2.0\n\n# Unless required by applicable law or agreed to in writing, software distributed\n# under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n# CONDITIONS OF ANY KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations under the License.\n\n\"\"\"\n# author: Martin Gorner\n# twitter: @martin_gorner\n# modified: See--\n# modified from:\n# https://github.com/GoogleCloudPlatform/training-data-analyst/blob/master/courses/fast-and-lean-data-science/03_Flower_pictures_to_TFRecords.ipynb\n\"\"\"\nimport tensorflow as tf\nimport os\nfrom PIL import Image\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport argparse\nfrom concurrent.futures import ThreadPoolExecutor\n\n\ndef read_image_label(inputs):\n  img_bytes = tf.io.read_file(inputs['img'])\n  return img_bytes, inputs['image_id'], inputs['grapheme_root'], inputs['vowel_diacritic'], \\\n      inputs['consonant_diacritic'], inputs['unique_tuple']\n\n\ndef to_tfrecord(img_bytes, image_id, grapheme_root, vowel_diacritic,\n      consonant_diacritic, unique_tuple):\n  feature = {\n      'img': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_bytes])),\n      'image_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[image_id])),\n      'grapheme_root': tf.train.Feature(int64_list=tf.train.Int64List(value=[grapheme_root])),\n      'vowel_diacritic': tf.train.Feature(int64_list=tf.train.Int64List(value=[vowel_diacritic])),\n      'consonant_diacritic': tf.train.Feature(int64_list=tf.train.Int64List(value=[\n          consonant_diacritic])),\n      'unique_tuple': tf.train.Feature(int64_list=tf.train.Int64List(value=[unique_tuple])),\n  }\n  return tf.train.Example(features=tf.train.Features(feature=feature))\n\n\ndef get_img_size(fn):\n  try:\n    # width, height = im.size\n    img_size = Image.open(fn).size[::-1]\n\n  except Exception as e:\n    print(f'{fn} errored with {e}')\n    img_size = None\n  return img_size\n\n\ndef main():\n  parser = argparse.ArgumentParser()\n  parser.add_argument('--clean', action='store_true')\n  parser.add_argument('--version', type=str, default='v0.1.0')\n  parser.add_argument('--do_not_train', action='store_true')\n  parser.add_argument('--records_dir', type=str, default='records')\n  parser.add_argument('--image_glob', type=str, default='images/*.png')\n  parser.add_argument('--seed', type=int, default=123)\n  args, _ = parser.parse_known_args()\n\n  np.random.seed(args.seed)\n  os.makedirs(args.records_dir, exist_ok=True)\n  if args.clean:\n    os.system(f'rm -f {args.records_dir}/*.tfrec')\n    print('Done cleaning')\n    return 0\n\n  fns = sorted(tf.io.gfile.glob(args.image_glob),\n      key=lambda x: int(x.split('_')[1]))\n  perm = np.random.permutation(len(fns))\n  vaild_len = int(0.1 * len(fns))\n  val_fns = [fns[p] for p in perm[:vaild_len]]\n  print(\"val_len:{}\".format(len(perm[:vaild_len])))\n  index = [i for i in range(0,vaild_len)]\n  perm = np.delete(perm,index)\n  print(\"train_len:{}\".format(len(perm)))\n  train_fns = [fns[p] for p in perm]\n  \n  print(f'{len(train_fns)} training and {len(val_fns)} validation fns')\n  num_shards = 1\n  for prefix in ['val', 'train']:\n    if prefix == 'train' and args.do_not_train:\n      continue\n\n    if prefix == 'train':\n      img_filenames = train_fns\n    else:\n      img_filenames = val_fns\n\n    print('Removing images with bad shape')\n    # remove images with bad shape\n    with ThreadPoolExecutor() as e:\n      img_sizes = list(tqdm(e.map(get_img_size, img_filenames), total=len(\n        img_filenames)))\n\n    img_sizes = [tf.constant(sz, tf.int64) for sz in img_sizes]\n\n    # e.g: 'images/Train_116991_rt_53_vl_7_ct_4_ti_343.png'\n    #       000000000000_111111_22_33_44_5_66_7_88_9999999\n    image_id = [int(fn.split('_')[1]) for fn in img_filenames]\n    grapheme_root = [int(fn.split('_')[3]) for fn in img_filenames]\n    vowel_diacritic = [int(fn.split('_')[5]) for fn in img_filenames]\n    consonant_diacritic = [int(fn.split('_')[7]) for fn in img_filenames]\n    unique_tuple = [int(fn.split('_')[9][:-4]) for fn in img_filenames]\n\n    if prefix == 'train':\n      num_shards = 10\n    else:\n      num_shards = 2\n\n    ds = tf.data.Dataset.from_tensor_slices({'img': img_filenames, 'image_id': image_id,\n        'grapheme_root': grapheme_root, 'vowel_diacritic': vowel_diacritic,\n        'consonant_diacritic': consonant_diacritic, 'unique_tuple': unique_tuple})\n    ds = ds.map(read_image_label)\n    ds = ds.batch(len(img_filenames) // num_shards)\n    print(\"Writing TFRecords\")\n    for shard_index, ret in tqdm(enumerate(ds), total=num_shards):\n      # batch size used as shard size here\n      img, image_id, r, v, c, ti = map(lambda x: x.numpy(), ret)\n      current_shard_size = img.shape[0]\n      # good practice to have the number of records in the filename\n      filename = os.path.join(args.records_dir, 'HALF_%s_%04d_%06d_%s.tfrec' % (\n          prefix, shard_index, current_shard_size, args.version))\n      with tf.io.TFRecordWriter(filename) as out_file:\n        for i in tqdm(range(current_shard_size)):\n          example = to_tfrecord(img[i], image_id[i], r[i], v[i], c[i], ti[i])\n          out_file.write(example.SerializeToString())\n        print(\"Wrote file {} containing {} records\".format(filename, current_shard_size))\n\n\nmain()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!du -sh images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls records","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}