{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello Fellow Kagglers,\n\nThis notebook demonstrates how to create TFRecords out of the extra training data crawled in [this](https://www.kaggle.com/markwijkhuizen/google-landmark-recognition-extra-data-tfrec-pub) notebook, using [this](https://www.kaggle.com/markwijkhuizen/google-landmark-recognition-extra-train-data-pub) dataset created by that notebook. This dataset contains images of classes with a low amount of samples. Using the complete [Google Landmarks Dataset v2](https://github.com/cvdfoundation/google-landmark) dataset, containing over 4 million images, all classes are filled up to 25 samples. This method is further explained in the notebook and resulted in over 400,000 training samples of classes with few samples.\n\nTFRecords are a highly efficient way to read many small files, such as JPEG's. Instead of reading many small images a single TFRecord containing many images can be read at once. Moreover, each record inside a TFRecords can contain additional data, such as the label. Using this TFRecord format several thousands of images per second can be read on a TPU.\n\nThe TFRecords dataset produced in this notebook can be found [here](https://www.kaggle.com/markwijkhuizen/google-landmark-recognition-extra-train-tfrecs-pub).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom kaggle_datasets import KaggleDatasets\nfrom multiprocessing import cpu_count\n\nimport sys\nimport cv2\nimport imageio\nimport joblib\nimport pickle\n\n# Activate pandas progress apply bar\ntqdm.pandas()\n\nprint(f'tensorflow version: {tf.__version__}')\nprint(f'tensorflow keras version: {tf.keras.__version__}')\nprint(f'python version: P{sys.version}')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:28.819129Z","iopub.execute_input":"2021-09-19T11:08:28.819993Z","iopub.status.idle":"2021-09-19T11:08:28.828491Z","shell.execute_reply.started":"2021-09-19T11:08:28.819955Z","shell.execute_reply":"2021-09-19T11:08:28.827667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Smaller side of the image, can be adjusted\nIMG_SIZE = 384\nN_CHANNELS = 3\nVERSION = '1A'","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:28.829863Z","iopub.execute_input":"2021-09-19T11:08:28.830392Z","iopub.status.idle":"2021-09-19T11:08:28.839886Z","shell.execute_reply.started":"2021-09-19T11:08:28.83036Z","shell.execute_reply":"2021-09-19T11:08:28.838838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_pickle('/kaggle/input/google-landmark-recognition-extra-train-data-pub/train_extra.pkl.xz')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:28.841759Z","iopub.execute_input":"2021-09-19T11:08:28.842293Z","iopub.status.idle":"2021-09-19T11:08:29.380488Z","shell.execute_reply.started":"2021-09-19T11:08:28.842261Z","shell.execute_reply":"2021-09-19T11:08:29.379357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head())","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.382241Z","iopub.execute_input":"2021-09-19T11:08:29.382544Z","iopub.status.idle":"2021-09-19T11:08:29.411196Z","shell.execute_reply.started":"2021-09-19T11:08:29.382515Z","shell.execute_reply":"2021-09-19T11:08:29.410293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# There are 402,962 additional training images in the dataset\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.412467Z","iopub.execute_input":"2021-09-19T11:08:29.412764Z","iopub.status.idle":"2021-09-19T11:08:29.476519Z","shell.execute_reply.started":"2021-09-19T11:08:29.412734Z","shell.execute_reply":"2021-09-19T11:08:29.47559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Landmark ID to label mapper\nwith open('/kaggle/input/landmark-recognition-2021-tfrecords-384-part-1/landmark_id2label.pkl', 'rb') as f:\n    landmark_id2label = pickle.load(f)\n    \n# Add label to DataFrame\ntrain['label'] = train['landmark_id'].apply(landmark_id2label.get)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.477839Z","iopub.execute_input":"2021-09-19T11:08:29.478153Z","iopub.status.idle":"2021-09-19T11:08:29.900991Z","shell.execute_reply.started":"2021-09-19T11:08:29.478115Z","shell.execute_reply":"2021-09-19T11:08:29.899965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity check, all label fall in the range [0, 81312]\ntrain['label'].describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.902783Z","iopub.execute_input":"2021-09-19T11:08:29.903132Z","iopub.status.idle":"2021-09-19T11:08:29.932241Z","shell.execute_reply.started":"2021-09-19T11:08:29.903092Z","shell.execute_reply":"2021-09-19T11:08:29.931203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.head())","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.933701Z","iopub.execute_input":"2021-09-19T11:08:29.934007Z","iopub.status.idle":"2021-09-19T11:08:29.943964Z","shell.execute_reply.started":"2021-09-19T11:08:29.933978Z","shell.execute_reply":"2021-09-19T11:08:29.943292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.info())","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:29.946771Z","iopub.execute_input":"2021-09-19T11:08:29.947064Z","iopub.status.idle":"2021-09-19T11:08:30.008158Z","shell.execute_reply.started":"2021-09-19T11:08:29.947037Z","shell.execute_reply":"2021-09-19T11:08:30.007112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# File Path","metadata":{}},{"cell_type":"code","source":"# Add File Path to Image\ndef to_file_path(i):\n    return f'/kaggle/input/google-landmark-recognition-extra-train-data-pub/train/{i[0]}/{i[0]}/{i[1]}/{i[2]}/{i}.jpg'\n\ntrain['file_path'] = train['id'].progress_apply(to_file_path).astype('string')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:47.402781Z","iopub.execute_input":"2021-09-19T11:08:47.403148Z","iopub.status.idle":"2021-09-19T11:08:48.707368Z","shell.execute_reply.started":"2021-09-19T11:08:47.403117Z","shell.execute_reply":"2021-09-19T11:08:48.706362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shuffle DataFrame\ntrain = train.sample(frac=1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:48.708761Z","iopub.execute_input":"2021-09-19T11:08:48.709076Z","iopub.status.idle":"2021-09-19T11:08:48.86381Z","shell.execute_reply.started":"2021-09-19T11:08:48.709042Z","shell.execute_reply":"2021-09-19T11:08:48.862795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Image","metadata":{}},{"cell_type":"code","source":"def process_image(file_path):\n    # Read Image\n    img = imageio.imread(file_path)\n    h, w, _ = img.shape\n\n    r = IMG_SIZE / min(w, h)\n    # Check whether image is bigger than IMG_SIZE\n    if min(h,w) > IMG_SIZE:\n        w_resize = int(w * r)\n        h_resize = int(h * r)\n        # Resize using high quality LANCZOS algorithm\n        img = cv2.resize(img, (w_resize, h_resize), interpolation=cv2.INTER_LANCZOS4)\n        # Save as JPEG with quality set to 70, just as original images\n        img_jpeg = tf.io.encode_jpeg(img, quality=70, optimize_size=True).numpy()\n        return img_jpeg, h_resize, w_resize\n    # Otherwise use original image\n    else:\n        with open(file_path, 'rb') as f:\n            img_jpeg = f.read()\n        return img_jpeg, h, w","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:48.865716Z","iopub.execute_input":"2021-09-19T11:08:48.86601Z","iopub.status.idle":"2021-09-19T11:08:48.873808Z","shell.execute_reply.started":"2021-09-19T11:08:48.865982Z","shell.execute_reply":"2021-09-19T11:08:48.872925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split in Chunks","metadata":{}},{"cell_type":"code","source":"# Split Training Data in Chunks\ndef split_in_chunks(data, chunk_size):\n    return [data[:, i:i + CHUNK_SIZE] for i in range(0, len(data[1]), CHUNK_SIZE)]","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:49.137319Z","iopub.execute_input":"2021-09-19T11:08:49.137683Z","iopub.status.idle":"2021-09-19T11:08:49.142517Z","shell.execute_reply.started":"2021-09-19T11:08:49.13765Z","shell.execute_reply":"2021-09-19T11:08:49.141697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each TFRecords will yield 3000 images\nCHUNK_SIZE = int(3e3)\n\n# Split train data in chunks\ntrain_split = split_in_chunks(np.array((train['file_path'], train['label'])), CHUNK_SIZE)\n\nprint(f'train_split chunks: {len(train_split)}')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:49.566809Z","iopub.execute_input":"2021-09-19T11:08:49.567193Z","iopub.status.idle":"2021-09-19T11:08:49.599807Z","shell.execute_reply.started":"2021-09-19T11:08:49.567164Z","shell.execute_reply":"2021-09-19T11:08:49.599094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make TFRecords","metadata":{}},{"cell_type":"code","source":"# Makes the actual TFRecords\ndef to_tf_records(data_split, name):\n    for idx, (fps, lbls) in enumerate(tqdm(data_split)):\n        \n        # Create image processing jobs and execute them in parallel\n        jobs = [joblib.delayed(process_image)(fp) for fp in fps]\n        imgs_resized = joblib.Parallel(\n            n_jobs=cpu_count(),\n            verbose=0,\n            batch_size=64,\n            pre_dispatch=64*cpu_count(),\n            require='sharedmem'\n        )(jobs)\n        tfrecord_name = f'{VERSION}_EXTRA_DATA_{name}_batch_{idx}.tfrecords'\n        \n        # Create the actual TFRecords\n        with tf.io.TFRecordWriter(tfrecord_name) as file_writer:\n            for (img, h, w), lbl in zip(imgs_resized, lbls):\n                record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                    # Image as JPEG bytes\n                    'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img])),\n                    # Label of image\n                    'label': tf.train.Feature(int64_list=tf.train.Int64List(value=[int(lbl)])),\n                    # Height of image\n                    'height': tf.train.Feature(int64_list=tf.train.Int64List(value=[int(h)])),\n                    # Width of image\n                    'width': tf.train.Feature(int64_list=tf.train.Int64List(value=[int(w)])),\n                })).SerializeToString()\n                file_writer.write(record_bytes)\n\n# Create TFRecords\nto_tf_records(train_split, 'train')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:08:56.831907Z","iopub.execute_input":"2021-09-19T11:08:56.832276Z","iopub.status.idle":"2021-09-19T11:09:27.012668Z","shell.execute_reply.started":"2021-09-19T11:08:56.83224Z","shell.execute_reply":"2021-09-19T11:09:27.011419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check TFRecords","metadata":{}},{"cell_type":"code","source":"# Imagenet mean and standard deviation per channel\nIMAGENET_MEAN = tf.constant([0.485, 0.456, 0.406], dtype=tf.float32)\nIMAGENET_STD = tf.constant([0.229, 0.224, 0.225], dtype=tf.float32)\n\n# Number of channels, 3 for RGB images\nN_CHANNELS = tf.constant(3, dtype=tf.int64)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:09:27.014433Z","iopub.execute_input":"2021-09-19T11:09:27.01484Z","iopub.status.idle":"2021-09-19T11:09:27.038203Z","shell.execute_reply.started":"2021-09-19T11:09:27.014799Z","shell.execute_reply":"2021-09-19T11:09:27.0374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to decode the TFRecords\ndef decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.int64),\n        'width': tf.io.FixedLenFeature([], tf.int64),\n        'height': tf.io.FixedLenFeature([], tf.int64),\n    })\n\n    image = tf.io.decode_jpeg(features['image'])\n    label = features['label']\n    height = features['height']\n    width = features['width']\n    \n    # Cutout Random Square if image is not square\n    if height != width:\n        if height > width:\n            offset = tf.random.uniform(shape=(), minval=0, maxval=height-width, dtype=tf.int64)\n            image = tf.slice(image, [offset, 0, 0], [width, width, N_CHANNELS])\n        else:\n            offset = tf.random.uniform(shape=(), minval=0, maxval=width-height, dtype=tf.int64)\n            image = tf.slice(image, [0, offset, 0], [height, height, N_CHANNELS])\n    \n    # Reshape and Normalize\n    size = tf.math.reduce_min([height, width])\n    # Explicit reshape needed for TPU, tell cimpiler dimensions of image\n    image = tf.reshape(image, [size, size, N_CHANNELS])\n    # Some images are smaller than 384x384 and need to be upscaled\n    image = tf.image.resize(image, [IMG_SIZE, IMG_SIZE])\n    # Convert to float32 and normalize to range 0-1\n    image = tf.cast(image, tf.float32)  / 255.0\n    # Normalize according to ImageNet mean and standard deviation\n    image = (image - IMAGENET_MEAN) / IMAGENET_STD\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:09:27.039757Z","iopub.execute_input":"2021-09-19T11:09:27.040135Z","iopub.status.idle":"2021-09-19T11:09:27.053305Z","shell.execute_reply.started":"2021-09-19T11:09:27.040107Z","shell.execute_reply":"2021-09-19T11:09:27.052488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shows a batch of images\ndef show_batch(dataset, rows=4, cols=3):\n    imgs, lbls = next(iter(dataset))\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*4, rows*4))\n    for r in range(rows):\n        for c in range(cols):\n            img = imgs[r*cols+c].numpy().astype(np.float32)\n            img += abs(img.min())\n            img /= img.max()\n            axes[r, c].imshow(img)\n            axes[r, c].set_title(f'Label: {lbls[r*cols+c]}')","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:09:27.054426Z","iopub.execute_input":"2021-09-19T11:09:27.054804Z","iopub.status.idle":"2021-09-19T11:09:27.072397Z","shell.execute_reply.started":"2021-09-19T11:09:27.054777Z","shell.execute_reply":"2021-09-19T11:09:27.071274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Makes a TFRecordDataser iterator\ndef get_train_dataset():\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob('./*.tfrecords')\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1)\n    train_dataset = train_dataset.map(decode_tfrecord, num_parallel_calls=1)\n    train_dataset = train_dataset.batch(32)\n    \n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:09:27.07411Z","iopub.execute_input":"2021-09-19T11:09:27.074807Z","iopub.status.idle":"2021-09-19T11:09:27.086834Z","shell.execute_reply.started":"2021-09-19T11:09:27.074757Z","shell.execute_reply":"2021-09-19T11:09:27.085529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity Check, plot some images from the freshly created TFRecords\ntrain_dataset = get_train_dataset()\nshow_batch(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-09-19T11:09:27.08843Z","iopub.execute_input":"2021-09-19T11:09:27.088782Z","iopub.status.idle":"2021-09-19T11:09:30.417455Z","shell.execute_reply.started":"2021-09-19T11:09:27.088751Z","shell.execute_reply":"2021-09-19T11:09:30.415676Z"},"trusted":true},"execution_count":null,"outputs":[]}]}