{"cells": [{"metadata": {"_uuid": "f0ff0fa58ebd4e88ec31e01195932a4812520659", "_cell_guid": "11a1f1d5-61a8-4ec5-a3d6-d22e2f31bd8a"}, "cell_type": "markdown", "source": ["This notebook showcases a fast way to read data from BSON into a generator for Keras.  \n", "The idea is strongly inspired from https://www.kaggle.com/humananalog/keras-generator-for-reading-directly-from-bson  \n", "\n", "Since I don't have a SSD, the original Generator is ~3s per batch , I just added the feature to read chunks of file and shuffle into batch to improve speed."]}, {"metadata": {"_uuid": "fc38b9e68ba0d3983c73c18ea7abb0142131068a", "ExecuteTime": {"end_time": "2017-10-07T20:46:30.16598Z", "start_time": "2017-10-07T20:46:29.715411Z"}, "_cell_guid": "3e302aca-bdfb-4792-b2d0-a59785d822a8", "collapsed": true}, "source": ["import pandas as pd\n", "import numpy as np"], "cell_type": "code", "outputs": [], "execution_count": 1}, {"metadata": {"_uuid": "a6168d40bff582e580e08e194c4754c694ee4764", "ExecuteTime": {"end_time": "2017-10-07T20:46:30.318898Z", "start_time": "2017-10-07T20:46:30.305183Z"}, "_cell_guid": "84535c0b-3d37-487d-a974-81d33582b39e", "collapsed": true}, "source": ["import bson"], "cell_type": "code", "outputs": [], "execution_count": 2}, {"metadata": {"_uuid": "78dd1494cff1ec8f5115d2c3164847320536c272", "ExecuteTime": {"end_time": "2017-10-07T21:05:39.264847Z", "start_time": "2017-10-07T21:05:39.110927Z"}, "_cell_guid": "b3ea5369-a1bb-48c1-8de9-84e056f90464", "collapsed": true}, "source": ["import cv2"], "cell_type": "code", "outputs": [], "execution_count": 3}, {"metadata": {"_uuid": "d41fbd83fa9625c5b5d06ed298ef36dfef48a111", "_cell_guid": "20a8050b-9a5b-4b1b-9636-a32d468e2d0e", "collapsed": true}, "source": ["from tqdm import *\n", "import struct\n"], "cell_type": "code", "outputs": [], "execution_count": 4}, {"metadata": {"_uuid": "727b6f8bb15f15c9e5f5948e4424979f534e291b", "_cell_guid": "9f7c2825-463a-4951-9423-a9cc5a308fd5"}, "cell_type": "markdown", "source": ["# Load meta data"]}, {"metadata": {"_uuid": "ba2a2d2e959c961490f4ff47e3eb17105eb55cd1", "_cell_guid": "f175ac0a-76fa-45dc-b740-0656296ed4b7"}, "cell_type": "markdown", "source": ["From https://www.kaggle.com/humananalog/keras-generator-for-reading-directly-from-bson"]}, {"metadata": {"_uuid": "3f076205f2bf977b286c4c7bf46e26f2a2e5d8fd", "_cell_guid": "6f57d50f-58c9-4e10-a9e6-f8918e9a6e2d", "collapsed": true}, "source": ["def read_bson(bson_path, num_records, with_categories):\n", "    rows = {}\n", "    with open(bson_path, \"rb\") as f, tqdm(total=num_records) as pbar:\n", "        offset = 0\n", "        while True:\n", "            item_length_bytes = f.read(4)\n", "            if len(item_length_bytes) == 0:\n", "                break\n", "\n", "            length = struct.unpack(\"<i\", item_length_bytes)[0]\n", "\n", "            f.seek(offset)\n", "            item_data = f.read(length)\n", "            assert len(item_data) == length\n", "\n", "            item = bson.BSON.decode(item_data)\n", "            product_id = item[\"_id\"]\n", "            num_imgs = len(item[\"imgs\"])\n", "\n", "            row = [num_imgs, offset, length]\n", "            if with_categories:\n", "                row += [item[\"category_id\"]]\n", "            rows[product_id] = row\n", "\n", "            offset += length\n", "            f.seek(offset)\n", "            pbar.update()\n", "\n", "    columns = [\"num_imgs\", \"offset\", \"length\"]\n", "    if with_categories:\n", "        columns += [\"category_id\"]\n", "\n", "    df = pd.DataFrame.from_dict(rows, orient=\"index\")\n", "    df.index.name = \"product_id\"\n", "    df.columns = columns\n", "    df.sort_index(inplace=True)\n", "    return df"], "cell_type": "code", "outputs": [], "execution_count": 5}, {"metadata": {"_uuid": "87714d740b8dbc920146c287ce41c7672b4943bf", "ExecuteTime": {"end_time": "2017-10-07T20:46:31.991827Z", "start_time": "2017-10-07T20:46:31.987515Z"}, "_cell_guid": "efa1e835-8ba5-4a30-a5ee-a32174639339", "collapsed": true}, "source": ["TRAIN_BSON_FILE = '../input/train.bson'"], "cell_type": "code", "outputs": [], "execution_count": 6}, {"metadata": {"_uuid": "097bae71e52f4d7fb52211012d442e21735bbafe", "ExecuteTime": {"end_time": "2017-10-07T20:46:41.355685Z", "start_time": "2017-10-07T20:46:33.539742Z"}, "_cell_guid": "208c31fc-d8ca-49ca-af3f-06c3962b1192"}, "source": ["meta_data = read_bson(TRAIN_BSON_FILE, 7069896, with_categories=True)"], "cell_type": "code", "outputs": [], "execution_count": 7}, {"metadata": {"_uuid": "358d39d2656ddb6f2441047e5f3375ba73290193", "_cell_guid": "c9b4b6c1-59d6-4383-854e-c1d4863469b3"}, "cell_type": "markdown", "source": ["# Test read performance"]}, {"metadata": {"_uuid": "973c3ac8d352e923068b134e5ac87711ff817de8", "_cell_guid": "e68e07d7-0935-4f00-a74d-b4910502c847"}, "cell_type": "markdown", "source": ["The read performance on HDD is orders pr magnitude higher when random read vs contiguous read"]}, {"metadata": {"_uuid": "e2f8124ac155e4f704088bb7193a753d24adb12c", "ExecuteTime": {"end_time": "2017-10-07T20:47:03.623339Z", "start_time": "2017-10-07T20:47:03.618258Z"}, "_cell_guid": "7d1e01b5-fb75-4da6-807e-9e068263abce", "collapsed": true}, "source": ["def get_obs(file, offset, length):\n", "    file.seek(offset)\n", "    return bson.BSON.decode(file.read(length))"], "cell_type": "code", "outputs": [], "execution_count": 8}, {"metadata": {"_uuid": "1b42d295da62199489e52c6cb2819431b7ecb5f1", "ExecuteTime": {"end_time": "2017-10-07T20:47:04.103879Z", "start_time": "2017-10-07T20:47:04.100248Z"}, "_cell_guid": "6519f6d9-f373-4dde-9649-0a2ca5af842a", "collapsed": true}, "source": ["file = open(TRAIN_BSON_FILE, 'rb')"], "cell_type": "code", "outputs": [], "execution_count": 9}, {"metadata": {"_uuid": "f95c9d06d40384e615488536965fb0d4c127c204", "ExecuteTime": {"end_time": "2017-10-07T09:04:52.08141Z", "start_time": "2017-10-07T09:04:52.077292Z"}, "_cell_guid": "3fa75050-d4f4-4715-a281-ff225306385f"}, "cell_type": "markdown", "source": ["##\u00a0Contiguous read"]}, {"metadata": {"_uuid": "390f2122a868f46712e4f91c05e51e3b75f7755c", "ExecuteTime": {"end_time": "2017-10-07T20:47:07.436598Z", "start_time": "2017-10-07T20:47:05.538698Z"}, "_cell_guid": "00779bc4-18cf-4b5d-a558-bad3ffdb1147"}, "source": ["%%timeit\n", "i = np.random.choice(meta_data.shape[0], size=1)[0]\n", "sample = meta_data.iloc[i:i+256]\n", "res = []\n", "for _id, row in sample.iterrows():\n", "    obs = get_obs(file, row['offset'], row['length'])\n", "    assert _id == obs['_id'] "], "cell_type": "code", "outputs": [], "execution_count": 10}, {"metadata": {"_uuid": "ea8d9511de5db7656d8677e759da69d24cd88e00", "ExecuteTime": {"end_time": "2017-10-07T09:07:16.449651Z", "start_time": "2017-10-07T09:07:16.39573Z"}, "_cell_guid": "11c014a3-5ed9-44da-8a4b-7970e4e5fe64"}, "cell_type": "markdown", "source": ["##\u00a0Random read"]}, {"metadata": {"_uuid": "10003b2cf8169af9d724e0f8862543b43acb39a9", "ExecuteTime": {"end_time": "2017-10-07T20:47:21.062938Z", "start_time": "2017-10-07T20:47:07.438296Z"}, "_cell_guid": "b36bf6c3-1141-40a6-97d3-e699ee6634a6"}, "source": ["%%timeit\n", "sample = meta_data.sample(256)\n", "for _id, row in sample.iterrows():\n", "    obs = get_obs(file, row['offset'], row['length'])\n", "    assert _id == obs['_id']"], "cell_type": "code", "outputs": [], "execution_count": 11}, {"metadata": {"_uuid": "cebfefda532acbbf73a0ed86b43e860dfc3f9aea", "_cell_guid": "74c81312-7b4d-43c6-94b7-5921a350a99b"}, "cell_type": "markdown", "source": ["It takes ~600 ms here but on my hdd it is 3.15 s per loop. With more preprocessing of the batch, my GPU would be starving all the time"]}, {"metadata": {"_uuid": "6d8b28d59f3a33e4af65880e73aea0e192e22842", "_cell_guid": "73f2c2b1-3d06-4000-8a65-6792909d90e4"}, "cell_type": "markdown", "source": ["##\u00a0Semi contiguous read"]}, {"metadata": {"_uuid": "1de96d44e3a73a5a558358cc13466ce5a328ccef", "_cell_guid": "c48151dd-997d-4793-a637-9a6b1fd78951"}, "cell_type": "markdown", "source": ["Here we can simulate getting a chunk and read only a sample from it. Since the offset are nearby in the file and sorted, it is still quite fast"]}, {"metadata": {"_uuid": "3ae0ac522c4080db4de43e77c9186bc42508d883", "ExecuteTime": {"end_time": "2017-10-07T20:47:22.950743Z", "start_time": "2017-10-07T20:47:21.065802Z"}, "_cell_guid": "929d9bca-7895-4198-b9ac-159740d8f6a3"}, "source": ["%%timeit\n", "i = np.random.choice(meta_data.shape[0], size=1)[0]\n", "sample = meta_data.iloc[i:i+10000].sample(256).sort_index()\n", "res = []\n", "for _id, row in sample.iterrows():\n", "    obs = get_obs(file, row['offset'], row['length'])\n", "    assert _id == obs['_id'] "], "cell_type": "code", "outputs": [], "execution_count": 12}, {"metadata": {"_uuid": "2ae23a267d01ef9424e0fe9c5341ba9789b3584e", "_cell_guid": "faf1930b-ebf8-4971-b9c4-0c5ec34c1a6b"}, "cell_type": "markdown", "source": ["##\u00a0Semi contiguous read wo  sort"]}, {"metadata": {"_uuid": "e4a70d12560263364bcb2e8727ff64505e8ad6b2", "ExecuteTime": {"end_time": "2017-10-07T20:47:45.778131Z", "start_time": "2017-10-07T20:47:40.146431Z"}, "_cell_guid": "2841d88b-eb18-4d03-95f4-91247b5d900e"}, "source": ["%%timeit\n", "i = np.random.choice(meta_data.shape[0], size=1)[0]\n", "sample = meta_data.iloc[i:i+10000].sample(256)\n", "res = []\n", "for _id, row in sample.iterrows():\n", "    obs = get_obs(file, row['offset'], row['length'])\n", "    assert _id == obs['_id'] "], "cell_type": "code", "outputs": [], "execution_count": 13}, {"metadata": {"_uuid": "a15f090a9d83bff3f9aa32b0adf288cc4b5597f4", "_cell_guid": "0d4e9717-9312-477e-9f91-2c5203c4877d"}, "cell_type": "markdown", "source": ["# Generator"]}, {"metadata": {"_uuid": "476a37bbd10108a096ebb3f64f87e9c86285d808", "ExecuteTime": {"end_time": "2017-10-07T21:06:21.788944Z", "start_time": "2017-10-07T21:06:21.782798Z"}, "_cell_guid": "ab5411d5-61a5-4e38-acbb-91fb6edd9c24"}, "source": ["from keras.preprocessing.image import Iterator\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras import backend as K\n", "from keras.applications import inception_v3"], "cell_type": "code", "outputs": [], "execution_count": 14}, {"metadata": {"_uuid": "8cd75efd7e3f8dcd6a604be8d6261bf35a1857db", "ExecuteTime": {"end_time": "2017-10-07T20:48:56.79379Z", "start_time": "2017-10-07T20:48:56.78926Z"}, "_cell_guid": "8c9e1f4b-c068-41b3-a894-8c56499b903d", "collapsed": true}, "source": ["from threading import Lock"], "cell_type": "code", "outputs": [], "execution_count": 15}, {"metadata": {"_uuid": "976ce1c80751bc79e8fb710388193c850b5e4de7", "_cell_guid": "34fd2e30-4da0-46d0-a64f-9f4be5be1b25"}, "cell_type": "markdown", "source": ["** Let's define 2 mode of read: contiguous read for test (faster) and block random read for train (slower but not too much) **"]}, {"metadata": {"_uuid": "b5157af35dd2b3f5166d41e83882ab3d89d29926", "ExecuteTime": {"end_time": "2017-10-07T20:47:45.794497Z", "start_time": "2017-10-07T20:47:45.780762Z"}, "_cell_guid": "50d5260b-c4a9-44b4-8f70-d81ae9008c36", "collapsed": true}, "source": ["def contiguous_read(bson_file):\n", "    while True:\n", "        iter_file = bson.decode_file_iter(open(bson_file, 'rb'))\n", "        for obs in iter_file:\n", "            yield obs"], "cell_type": "code", "outputs": [], "execution_count": 16}, {"metadata": {"_uuid": "d2b3510382e2d5c1065aa2ef6ec70bf362c4edc5", "_cell_guid": "0942eb67-92aa-40f9-af05-d9ea75a25e95"}, "cell_type": "markdown", "source": ["The idea is to split file into chunks of rather small size and read chunk in shuffled order"]}, {"metadata": {"_uuid": "3df01533d8e56e40983f7d0eed7cdca4246044b7", "ExecuteTime": {"end_time": "2017-10-07T20:47:46.595088Z", "start_time": "2017-10-07T20:47:46.578634Z"}, "_cell_guid": "7f1e3f69-9924-4b68-a0cb-c27fa4bd3dcd", "collapsed": true}, "source": ["def block_reader(bson_file, meta_data, chunk_size=1000, shuffle=False):\n", "    assert isinstance(meta_data, pd.DataFrame)\n", "    assert len(meta_data.columns.intersection(['offset', 'length'])) == 2\n", "    # prepare metadata\n", "    n_obs = meta_data.shape[0]\n", "    meta_data_sorted = meta_data.sort_values('offset', ascending=True)\n", "    meta_chunks = np.array_split(meta_data_sorted, \n", "                                 np.ceil(n_obs/chunk_size))\n", "    open_file = open(bson_file, 'rb')\n", "    while True:\n", "        # Generate chunks order\n", "        chunk_indexes = np.arange(len(meta_chunks))\n", "        if shuffle:\n", "            chunk_indexes = np.random.permutation(chunk_indexes)\n", "        # Iterate over chunks\n", "        for ind in chunk_indexes:\n", "            chunk = meta_chunks[ind]\n", "            for _id, _meta in chunk.iterrows():\n", "                open_file.seek(_meta['offset'])\n", "                obs = bson.BSON.decode(open_file.read(_meta['length']))\n", "                yield obs"], "cell_type": "code", "outputs": [], "execution_count": 17}, {"metadata": {"_uuid": "68347f1266acae555616c601c743b3fb51e514b5", "_cell_guid": "1bc3a217-9e3e-4cdd-b9b7-b0a8440cfcf5"}, "cell_type": "markdown", "source": ["Chunk can still note shuffled inside, so we need a batching mecanism with some cache to shuffle more (inspired by tensorflow Iterator)"]}, {"metadata": {"_uuid": "81e45ebc8820347af4930189fb84d71f3789a3d3", "ExecuteTime": {"end_time": "2017-10-07T20:47:52.254334Z", "start_time": "2017-10-07T20:47:52.22912Z"}, "_cell_guid": "25694949-7639-444a-aa1c-fdc6d43f91aa", "collapsed": true}, "source": ["def batch(generator, batch_size, shuffle=False, cache_size=10000):\n", "    gen_stopped = False\n", "    cached_data = []\n", "    while True:        \n", "        # Fill up cache\n", "        while (len(cached_data) < cache_size) & (gen_stopped == False):\n", "            try:\n", "                cached_data.append(next(generator))\n", "            except StopIteration:\n", "                gen_stopped = True\n", "                break\n", "        # Stop if there is nothing left\n", "        if len(cached_data) == 0:\n", "            return\n", "        #\u00a0Generate batch       \n", "        batch_data = []\n", "        bsize = min(batch_size, len(cached_data))\n", "        if shuffle:\n", "            inds = np.random.choice(len(cached_data), size=(bsize,), replace=False)\n", "        else:\n", "            inds = np.arange(bsize)\n", "        \n", "        for i in sorted(inds, reverse=True):\n", "            try:\n", "                batch_data.append(cached_data.pop(i))            \n", "            except IndexError:\n", "                print(i, bsize, len(cached_data))\n", "        yield batch_data"], "cell_type": "code", "outputs": [], "execution_count": 18}, {"metadata": {"_uuid": "d92ddbe9e76cdf60da6e209bec487c2ca667b772", "_cell_guid": "3586de9d-f1dd-45f7-9c34-ad41dea31013"}, "cell_type": "markdown", "source": ["Some functions to process data"]}, {"metadata": {"_uuid": "0c06282b360f11f38d3e3a9d8b56159733bfa689", "ExecuteTime": {"end_time": "2017-10-07T20:47:53.182386Z", "start_time": "2017-10-07T20:47:53.171487Z"}, "_cell_guid": "3d9f9f39-1b91-4e69-a3b2-d17a7a5b9eee", "collapsed": true}, "source": ["def get_img(obs, img_size=180, keep=None):\n", "    if keep is None:\n", "        keep = np.random.choice(len(obs['imgs']))\n", "    else:\n", "        keep = 0\n", "    byte_str = obs['imgs'][keep]['picture']\n", "    img = cv2.imdecode(np.fromstring(byte_str, dtype=np.uint8), \n", "                       cv2.IMREAD_COLOR)\n", "    img = cv2.resize(img, (img_size,img_size))\n", "    return img"], "cell_type": "code", "outputs": [], "execution_count": 19}, {"metadata": {"_uuid": "ea162fc7a1743811d16dee19c8bd7589efe1b35e", "ExecuteTime": {"end_time": "2017-10-07T20:47:53.626342Z", "start_time": "2017-10-07T20:47:53.615729Z"}, "_cell_guid": "5c537134-5b55-495e-a54b-f6d4c9b367f7", "collapsed": true}, "source": ["def preprocess_batch(batch_data, labels=None, weights=None, img_size=180):\n", "    batch_size = len(batch_data)\n", "    X = np.zeros(shape=(batch_size, img_size, img_size, 3), dtype=np.float32)\n", "    y = np.zeros(shape=(batch_size,), dtype=np.float32)\n", "    w = np.ones(shape=(batch_size,), dtype=np.float32)\n", "    for ind, obs in enumerate(batch_data):\n", "        _id = obs['_id']\n", "        X[ind] = get_img(obs, img_size=img_size)\n", "        if labels is not None:\n", "            y[ind] = labels[_id]\n", "        if weights is not None:\n", "            w[ind] = weights[_id]\n", "    X = inception_v3.preprocess_input(X)\n", "    return X, y, w"], "cell_type": "code", "outputs": [], "execution_count": 20}, {"metadata": {"_uuid": "cbf99acf689d016a63489f66ab984ea80de90974", "_cell_guid": "2bfccade-bf2a-4018-8f0d-ff381fb547e2"}, "cell_type": "markdown", "source": ["** Let's wrap it up in a Iterator with lock so we can do multithreading later**"]}, {"metadata": {"_uuid": "feddd6c5068a2572a0666311c8d5ba324090df8e", "ExecuteTime": {"end_time": "2017-10-07T20:58:32.087698Z", "start_time": "2017-10-07T20:58:32.06088Z"}, "_cell_guid": "4027c326-03ad-4ca7-8080-9b12eb01b45f", "collapsed": true}, "source": ["class BSONIterator(Iterator):\n", "    def __init__(self, bson_file, batch_size=32, preprocess_batch_func=None, metadata=None,\n", "                 shuffle=False, chunk_size=1000, shuffle_cache=100000):\n", "        if shuffle:\n", "            self.obs_generator = block_reader(bson_file, metadata, chunk_size=chunk_size, shuffle=True)\n", "        else:\n", "            self.obs_generator = contiguous_read(bson_file)\n", "        self.batch_generator = batch(self.obs_generator, batch_size=batch_size, \n", "                                     shuffle=shuffle, cache_size=shuffle_cache)\n", "        self.preprocess_batch_func = preprocess_batch_func\n", "        self.lock = Lock()\n", "        \n", "    def next(self):\n", "        with self.lock:\n", "            batch_data = next(self.batch_generator)\n", "        if self.preprocess_batch_func is None:\n", "            return batch_data\n", "        else:\n", "            return self.preprocess_batch_func(batch_data)\n", "        \n", "    def __next__(self):\n", "        return self.next()"], "cell_type": "code", "outputs": [], "execution_count": 21}, {"metadata": {"_uuid": "bd343df17d837434fc1ad86165016dbe09eb17f8", "_cell_guid": "5ce692f7-79a3-4632-987b-832aa3f119f5"}, "cell_type": "markdown", "source": ["## Test"]}, {"metadata": {"_uuid": "40c29efd975bcfe63553cfa138ae843ba38fe9be", "ExecuteTime": {"end_time": "2017-10-07T20:55:57.637217Z", "start_time": "2017-10-07T20:55:57.629575Z"}, "_cell_guid": "302486fe-f361-4d04-b421-1003d0e8ab5a", "collapsed": true}, "source": ["from tqdm import *"], "cell_type": "code", "outputs": [], "execution_count": 22}, {"metadata": {"_uuid": "b06e4ffa74efc521ce859dc86d8482a0c79fd737", "_cell_guid": "4f668422-519f-4610-8246-99429b359d0d"}, "cell_type": "markdown", "source": ["##\u00a0contiguous read\n", "It is recommended to create a separate BSON file with validation data so you can read the whole file"]}, {"metadata": {"_uuid": "c1d917f71d868538be55e20435a602f5286885c3", "ExecuteTime": {"end_time": "2017-10-07T20:55:58.169891Z", "start_time": "2017-10-07T20:55:58.122007Z"}, "_cell_guid": "a755df79-98e3-4130-b032-1c3c77fff893"}, "source": ["gen = contiguous_read(TRAIN_BSON_FILE)\n", "for _ in tqdm(range(10000)):\n", "    obs = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 23}, {"metadata": {"_uuid": "2f6f91aff42704279aaeb136815d1a0a8cea9e79", "_cell_guid": "5acd2a42-8c01-4d68-9e20-0c8204dba7e1"}, "cell_type": "markdown", "source": ["##\u00a0Block read without shuffle"]}, {"metadata": {"_uuid": "78862f40b6cf8f1b804a43c71e6f87cebea3b93f", "ExecuteTime": {"end_time": "2017-10-07T20:56:01.791759Z", "start_time": "2017-10-07T20:55:59.083549Z"}, "_cell_guid": "fbe6211a-3de0-44a8-ad86-c6d99a03c1d5"}, "source": ["gen = block_reader(TRAIN_BSON_FILE, meta_data, chunk_size=1000, shuffle=False)\n", "next(gen) #warm up\n", "for _ in tqdm(range(10000)):\n", "    obs = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 24}, {"metadata": {"_uuid": "8dda9ffda2884beefa7ce5e6b0413f28d9187810", "_cell_guid": "1c7d4ead-800f-4db8-bf86-d9c5d967205d"}, "cell_type": "markdown", "source": ["##\u00a0Block read with shuffle\n", "The performance hit is hard but it is still much better pure random read"]}, {"metadata": {"_uuid": "7606415867044799ad47c339d7eb952308165564", "ExecuteTime": {"end_time": "2017-10-07T20:56:05.478867Z", "start_time": "2017-10-07T20:56:01.793286Z"}, "_cell_guid": "60d14600-24a9-40ff-8627-ffb3b952ff2a"}, "source": ["gen = block_reader(TRAIN_BSON_FILE, meta_data.sample(100000), chunk_size=1000, shuffle=True)\n", "next(gen) #warm up\n", "for _ in tqdm(range(10000)):\n", "    obs = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 25}, {"metadata": {"_uuid": "67d9c882409ee3d85a4452902cbd67d68cf8dcde", "_cell_guid": "22e94711-c1b9-495c-a7de-ab5823efd98c"}, "cell_type": "markdown", "source": ["## Let's try some more realistic settings"]}, {"metadata": {"_uuid": "75acef35b0329c961b3b8141210d3446faf43fe8", "_cell_guid": "4afb06bd-0fba-4969-96b7-dd2faa20fd43"}, "cell_type": "markdown", "source": ["** Testing settings: reading the whole file **"]}, {"metadata": {"_uuid": "8f69bbf18fc60f24cb4d7b721dd69b45d22fb9c3", "ExecuteTime": {"end_time": "2017-10-07T21:03:38.609158Z", "start_time": "2017-10-07T21:03:14.426403Z"}, "_cell_guid": "e5dda4a6-164f-40c6-b428-be448ee61ec1"}, "source": ["gen = BSONIterator(TRAIN_BSON_FILE, batch_size=256)\n", "next(gen) #warm up\n", "for _ in tqdm(range(1000)):\n", "    batch_data = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 26}, {"metadata": {"_uuid": "7dced93641dbb4ac5957181cd3af56761a6de66d", "_cell_guid": "de24c711-87bb-4d7f-a108-68170b84d4ef"}, "cell_type": "markdown", "source": ["** Training settings: random read**"]}, {"metadata": {}, "source": ["# Let's simulate case where train is 90% of total obs\n", "train_meta = meta_data.sample(frac=0.9, replace=False)"], "cell_type": "code", "outputs": [], "execution_count": 28}, {"metadata": {"_uuid": "204bc082a50d97c62500ed446584d37ac0aa4762", "ExecuteTime": {"end_time": "2017-10-07T21:03:04.853611Z", "start_time": "2017-10-07T21:02:08.74589Z"}, "_cell_guid": "8657c8ff-1224-40fa-bcff-4efed28cfca0"}, "source": ["gen = BSONIterator(TRAIN_BSON_FILE, batch_size=256, shuffle=True, metadata=train_meta)\n", "next(gen) #warm up\n", "for _ in tqdm(range(1000)):\n", "    batch_data = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 29}, {"metadata": {"_uuid": "4981353c6c4210d0aacca1a983332229356db9c3", "ExecuteTime": {"end_time": "2017-10-07T21:04:09.196526Z", "start_time": "2017-10-07T21:04:09.192452Z"}, "_cell_guid": "651cfa84-033a-4ae3-a80a-b57c159b67ec", "collapsed": true}, "source": ["import functools"], "cell_type": "code", "outputs": [], "execution_count": 30}, {"metadata": {"_uuid": "af2073389a629618035f4779f0685763bb42f374", "_cell_guid": "f23725e8-11ec-4798-b52b-b55c6b0abd7c"}, "cell_type": "markdown", "source": ["** Add preprocessing ** "]}, {"metadata": {"_uuid": "88b5e1c914e573fb0e7839ac7af9f0a484a82a43", "ExecuteTime": {"end_time": "2017-10-07T21:08:20.327506Z", "start_time": "2017-10-07T21:07:38.825847Z"}, "_cell_guid": "ffaff5d0-bd60-4054-9794-dfd69a554eeb"}, "source": ["gen = BSONIterator(TRAIN_BSON_FILE, batch_size=256, shuffle=True, metadata=train_meta, \n", "                   preprocess_batch_func=functools.partial(preprocess_batch, \n", "                                                           img_size=128, \n", "                                                           labels=meta_data.category_id))\n", "next(gen) #warm up\n", "for _ in tqdm(range(100)):\n", "    batch_data = next(gen)"], "cell_type": "code", "outputs": [], "execution_count": 31}, {"metadata": {"_uuid": "551511a0e0791e6c3a4d69cbdbc36283ddc84d18", "_cell_guid": "2a43a028-ceb2-4ebc-82ca-da5df50d327e"}, "cell_type": "markdown", "source": ["** Simulate (poorly) multithreading**"]}, {"metadata": {"_uuid": "fbbe77c53083cf76cbcf1f650ff672274906643a", "ExecuteTime": {"end_time": "2017-10-07T21:08:21.779233Z", "start_time": "2017-10-07T21:08:21.542171Z"}, "_cell_guid": "74997fa9-a472-4161-8a8c-0e5ba5903911", "collapsed": true}, "source": ["from sklearn.externals.joblib import Parallel, delayed"], "cell_type": "code", "outputs": [], "execution_count": 32}, {"metadata": {"_uuid": "41bc8e54c754e7ed5410b512f2785b405dc77eab", "ExecuteTime": {"end_time": "2017-10-07T21:11:37.163356Z", "start_time": "2017-10-07T21:11:02.590025Z"}, "_cell_guid": "eb5ce883-c41c-4820-b768-2227e21c560d"}, "source": ["gen = BSONIterator(TRAIN_BSON_FILE, batch_size=256, shuffle=True, metadata=train_meta, \n", "                   preprocess_batch_func=functools.partial(preprocess_batch, \n", "                                                           img_size=128, \n", "                                                           labels=meta_data.category_id))\n", "next(gen) #warm up\n", "_ = Parallel(n_jobs=4, backend='threading', verbose=1)(delayed(next)(gen) \n", "                                                   for _ in tqdm(range(100)))"], "cell_type": "code", "outputs": [], "execution_count": 33}, {"metadata": {}, "cell_type": "markdown", "source": ["5 batches/s should be enough especially most of the time my GPU can only hold batchsize of 128. "]}, {"metadata": {"_uuid": "1c546e94a92d009115f6e7b28f266744735254aa", "_cell_guid": "3ba58669-925b-4433-854e-5fd5ca4ab9a8"}, "cell_type": "markdown", "source": ["** DISCLAIMER ** : I did not actually test the shuffling capability but the BSON file seem already shuffled so it should do the job.  \n", "One can tune the chunk_size smaller and cache_size higher for more shuffling vs cost of memory and/or speed\n"]}, {"metadata": {"_uuid": "385d59db0e3b6444fb142f9a89f17c0de3579e5b", "_cell_guid": "2d1a22cd-d388-4c27-a72a-978bf9c52262"}, "cell_type": "markdown", "source": ["** Hope some might find this helpful. Enjoy GPU feeding! **"]}], "metadata": {"language_info": {"version": "3.6.1", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "nbconvert_exporter": "python", "name": "python"}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "toc": {"number_sections": true, "nav_menu": {}, "toc_window_display": false, "toc_position": {}, "sideBar": true, "toc_cell": false, "skip_h1_title": false, "toc_section_display": "block"}}, "nbformat_minor": 1, "nbformat": 4}