{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Exporting GLDv2-full test set to your GCS bucket\n\nNow that [late submission for Retrieval track is disalbed](https://www.kaggle.com/c/landmark-recognition-2020/discussion/180090), the only option for those who haven't participated in previous competition and/or haven't trained a proper global descriptor is to use the full GLDv2 dataset to test the models.\n\nOne can't directly download the whole dataset and apply the [`build_image_dataset.py`](https://github.com/tensorflow/models/blob/master/research/delf/delf/python/training/build_image_dataset.py) script to it to convert to TFRecords, because it is HUGE (the whole train set is 500GB). This notebook will allow you to download, convert the dataset to TFRecords, and upload to your GCS bucket with **as little as 2GB disk space**.\n\nBefore you use it:\n  - This notebook only for GLDv2 test for now. It can be modified to download the train set, but without train/validation splits.\n  - It works for **Google Colab**, NOT Kaggle!\n  - Modify the **TODO** parts with your credentials","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from google.colab import auth\nauth.authenticate_user()\n\nimport os\n\n# TODO: Enter your own Google Cloud Project name here\nos.environ[\"GCLOUD_PROJECT\"] = \"your-project-name\"\n!export GCLOUD_PROJECT=your-project-name","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"class FLAGS:\n    test_directory = 'test/images/*/*/*/'\n    # TODO: add path to your own GCS bucket folder here\n    output_directory = 'gs://your-gcs-bucket/folder'\n    test_csv_path = 'test.csv'\n    num_shards_per_part = 2 # per TAR file\n    seed = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport csv\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\n_FILE_IDS_KEY = 'file_ids'\n_IMAGE_PATHS_KEY = 'image_paths'\n_LABELS_KEY = 'labels'\n_TEST_SPLIT = 'test'\n_TRAIN_SPLIT = 'train'\n_VALIDATION_SPLIT = 'validation'\n\n\ndef _get_all_image_files_and_labels(name, csv_path, image_dir):\n  \"\"\"Process input and get the image file paths, image ids and the labels.\n\n  Args:\n    name: 'train' or 'test'.\n    csv_path: path to the Google-landmark Dataset csv Data Sources files.\n    image_dir: directory that stores downloaded images.\n  Returns:\n    image_paths: the paths to all images in the image_dir.\n    file_ids: the unique ids of images.\n    labels: the landmark id of all images. When name='test', the returned labels\n      will be an empty list.\n  Raises:\n    ValueError: if input name is not supported.\n  \"\"\"\n  image_paths = tf.io.gfile.glob(os.path.join(image_dir, '*.jpg'))\n  file_ids = [os.path.basename(os.path.normpath(f))[:-4] for f in image_paths]\n  if name == _TRAIN_SPLIT:\n    with tf.io.gfile.GFile(csv_path, 'rb') as csv_file:\n      df = pd.read_csv(csv_file)\n    df = df.set_index('id')\n    labels = [int(df.loc[fid]['landmark_id']) for fid in file_ids]\n  elif name == _TEST_SPLIT:\n    labels = []\n  else:\n    raise ValueError('Unsupported dataset split name: %s' % name)\n  return image_paths, file_ids, labels\n\n\ndef _process_image(filename):\n  \"\"\"Process a single image file.\n\n  Args:\n    filename: string, path to an image file e.g., '/path/to/example.jpg'.\n\n  Returns:\n    image_buffer: string, JPEG encoding of RGB image.\n    height: integer, image height in pixels.\n    width: integer, image width in pixels.\n  Raises:\n    ValueError: if parsed image has wrong number of dimensions or channels.\n  \"\"\"\n  # Read the image file.\n  with tf.io.gfile.GFile(filename, 'rb') as f:\n    image_data = f.read()\n\n  # Decode the RGB JPEG.\n  image = tf.io.decode_jpeg(image_data, channels=3)\n\n  # Check that image converted to RGB\n  if len(image.shape) != 3:\n    raise ValueError('The parsed image number of dimensions is not 3 but %d' %\n                     (image.shape))\n  height = image.shape[0]\n  width = image.shape[1]\n  if image.shape[2] != 3:\n    raise ValueError('The parsed image channels is not 3 but %d' %\n                     (image.shape[2]))\n\n  return image_data, height, width\n\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\n\ndef _convert_to_example(file_id, image_buffer, height, width, label=None):\n  \"\"\"Build an Example proto for the given inputs.\n\n  Args:\n    file_id: string, unique id of an image file, e.g., '97c0a12e07ae8dd5'.\n    image_buffer: string, JPEG encoding of RGB image.\n    height: integer, image height in pixels.\n    width: integer, image width in pixels.\n    label: integer, the landmark id and prediction label.\n\n  Returns:\n    Example proto.\n  \"\"\"\n  colorspace = 'RGB'\n  channels = 3\n  image_format = 'JPEG'\n  features = {\n      'image/height': _int64_feature(height),\n      'image/width': _int64_feature(width),\n      'image/colorspace': _bytes_feature(colorspace.encode('utf-8')),\n      'image/channels': _int64_feature(channels),\n      'image/format': _bytes_feature(image_format.encode('utf-8')),\n      'image/id': _bytes_feature(file_id.encode('utf-8')),\n      'image/encoded': _bytes_feature(image_buffer)\n  }\n  if label is not None:\n    features['image/class/label'] = _int64_feature(label)\n  example = tf.train.Example(features=tf.train.Features(feature=features))\n\n  return example\n\n\ndef _write_tfrecord(output_prefix, image_paths, file_ids, labels,\n                    part_idx, total_parts, num_shards_per_part):\n  \"\"\"Read image files and write image and label data into TFRecord files.\n\n  Args:\n    output_prefix: string, the prefix of output files, e.g. 'train'.\n    image_paths: list of strings, the paths to images to be converted.\n    file_ids: list of strings, the image unique ids.\n    labels: list of integers, the landmark ids of images. It is an empty list\n      when output_prefix='test'.\n\n  Raises:\n    ValueError: if the length of input images, ids and labels don't match\n  \"\"\"\n  if output_prefix == _TEST_SPLIT:\n    labels = [None] * len(image_paths)\n  if not len(image_paths) == len(file_ids) == len(labels):\n    raise ValueError('length of image_paths, file_ids, labels shoud be the' +\n                     ' same. But they are %d, %d, %d, respectively' %\n                     (len(image_paths), len(file_ids), len(labels)))\n\n  spacing = np.linspace(0, len(image_paths), num_shards_per_part + 1, dtype=np.int)\n\n  for shard in range(num_shards_per_part):\n    output_file = os.path.join(\n        FLAGS.output_directory,\n        '%s-%.5d-of-%.5d' % (output_prefix,\n                             part_idx * num_shards_per_part + shard,\n                             total_parts * num_shards_per_part))\n    writer = tf.io.TFRecordWriter(output_file)\n    print('    - Processing shard ', shard, ' and writing file ', output_file)\n    for i in range(spacing[shard], spacing[shard + 1]):\n      image_buffer, height, width = _process_image(image_paths[i])\n      example = _convert_to_example(file_ids[i], image_buffer, height, width,\n                                    labels[i])\n      writer.write(example.SerializeToString())\n    writer.close()\n\n\ndef _build_test_tfrecord_dataset(csv_path, image_dir, part_idx, total_parts):\n  \"\"\"Build a TFRecord dataset for the 'test' split.\n\n  Args:\n    csv_path: path to the 'test' Google-landmark Dataset csv Data Sources files.\n    image_dir: directory that stores downloaded images.\n\n  Returns:\n    Nothing. After the function call, sharded TFRecord files are materialized.\n  \"\"\"\n  image_paths, file_ids, labels = _get_all_image_files_and_labels(\n      _TEST_SPLIT, csv_path, image_dir)\n  _write_tfrecord(_TEST_SPLIT, image_paths, file_ids, labels,\n                  part_idx, total_parts, FLAGS.num_shards_per_part)\n\n\ndef build_test_dataset(part_idx, total_parts):\n  _build_test_tfrecord_dataset(FLAGS.test_csv_path, FLAGS.test_directory,\n                               part_idx, total_parts)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download TAR archive, extract, convert, delete, repeat\n\nThe dataset is already shuffled, so we're ok.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport hashlib\nfrom datetime import datetime\nimport subprocess\nimport shutil\nfrom urllib.request import urlretrieve\n\n\ndef download_to_gcs(split, indices):\n    csv_url = f'https://s3.amazonaws.com/google-landmark/metadata/{split}.csv'\n    print('  - Downloading csv {}...'.format(csv_url), end=' ')\n    begin = datetime.now()\n    urlretrieve(csv_url, f'{split}.csv')\n    print('Done in', datetime.now() - begin)\n    \n    tar_format_url = 'https://s3.amazonaws.com/google-landmark/{split}/images_{idx:03d}.tar'\n    md5_format_url = 'https://s3.amazonaws.com/google-landmark/md5sum/{split}/md5.images_{idx:03d}.txt'\n    \n    for idx in indices:\n        print(f'{split}-{idx:03d}:')\n        \n        archive_url = tar_format_url.format(split=split, idx=idx)\n        print('  - Downloading archive {}...'.format(archive_url), end=' ')\n        begin = datetime.now()\n        urlretrieve(archive_url, 'tmp_images.tar')\n        print('Done in', datetime.now() - begin)\n\n        checksum_url = md5_format_url.format(split=split, idx=idx)\n        print('  - Downloading md5 checksum {}...'.format(checksum_url), end=' ')\n        begin = datetime.now()\n        urlretrieve(checksum_url, 'tmp_md5.txt')\n        print('Done in', datetime.now() - begin)\n\n        print('  - Calculating MD5 checksum...', end=' ')\n        md5sum_call = subprocess.Popen(['md5sum', 'tmp_images.tar'], stdout=subprocess.PIPE)\n        md5sum, _ = md5sum_call.communicate()\n        md5sum_call.wait()\n        md5sum = md5sum.decode('ascii').split()[0]\n\n        with open('tmp_md5.txt', 'r') as f:\n            md5sum_expected = f.read().split()[0]\n\n        if not md5sum == md5sum_expected:\n            print('Mismatch, the index will be added to `failed_indices.txt')\n            raise ValueError\n        else:\n            print('Matched')\n\n        print('  - Extracting archive to images folder...', end=' ')\n        begin = datetime.now()\n        if not os.path.isdir(f'{split}/images/'):\n            os.makedirs(f'{split}/images/')\n        extraction_task = subprocess.Popen(['tar', '-xf', './tmp_images.tar', '-C', f'{split}/images/'])\n        extraction_task.wait()\n        print('Done in', datetime.now() - begin)\n\n        print('  - Building TFRecrod and extracting to GCS:')\n        begin = datetime.now()\n        if split == 'test':\n            build_test_dataset(idx, len(indices))\n        else:\n            raise ValueError('split should be test')\n        print('    Done in', datetime.now() - begin)\n        \n        print('  - Cleaning up working directory...', end=' ')\n        begin = datetime.now()\n        os.remove('tmp_images.tar')\n        os.remove('tmp_md5.txt')\n        shutil.rmtree(f'{split}/')\n        print('Done in', datetime.now() - begin)\n    \n    print('Finished successfully')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"download_to_gcs('test', [idx for idx in range(20)])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}