{"metadata": {"language_info": {"version": "3.6.3", "codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "nbconvert_exporter": "python", "file_extension": ".py"}, "kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}, "nbformat": 4, "cells": [{"metadata": {"_cell_guid": "879a821a-e9ae-41c6-ae8c-ba58f1b2ac46", "collapsed": true, "_uuid": "30c446458d989a97bd2472ba3ffdc8a38d6c6a9e"}, "cell_type": "code", "source": ["from tqdm import tqdm_notebook\n", "import bson\n", "import numpy as np\n", "import io\n", "import matplotlib.pyplot as plt\n", "from PIL import Image\n", "% matplotlib inline"], "outputs": [], "execution_count": null}, {"metadata": {"_cell_guid": "b58d361e-f8e8-4ff7-8536-868e608b4275", "collapsed": true, "_uuid": "449b3d3c3ed68a96dfc3a7a5196d444c72f8cc22"}, "cell_type": "code", "source": ["num_images = 12371293\n", "num_points = 1000"], "outputs": [], "execution_count": null}, {"metadata": {"_cell_guid": "20987afc-a63f-4288-900c-986e389a736d", "collapsed": true, "_uuid": "3f3edfd817ed34e7bcef2496b7d1c008d7583492"}, "cell_type": "code", "source": ["checkpoints = np.linspace(0, num_images, num_points, dtype=np.int32)\n", "file_pointers = [0]"], "outputs": [], "execution_count": null}, {"metadata": {"_cell_guid": "7c4aec86-441f-4b9f-93b3-6cbe0358326f", "_uuid": "00d0a24922646887d7b7dff910acd9815b146643"}, "cell_type": "code", "source": ["bar = tqdm_notebook(total=num_images)\n", "i = 0\n", "current_checkpoint = 0\n", "with open('../input/train.bson', 'rb') as fbson:\n", "    data = bson.decode_file_iter(fbson)\n", "\n", "    for c, d in enumerate(data):\n", "        category = d['category_id']\n", "        _id = d['_id']\n", "        for e, pic in enumerate(d['imgs']):\n", "            i += 1\n", "            bar.update()\n", "        \n", "        if i > checkpoints[current_checkpoint + 1] and i < checkpoints[current_checkpoint + 2]:\n", "            file_pointers.append(fbson.tell())\n", "            current_checkpoint += 1"], "outputs": [], "execution_count": null}, {"metadata": {"_cell_guid": "d4fec908-53fb-45c0-9ac1-4aa168acaa61", "collapsed": true, "_uuid": "cb5e08f62b089973165f3b0da8d322f1f69e530c"}, "cell_type": "code", "source": ["file_pointers"], "outputs": [], "execution_count": null}, {"metadata": {"_cell_guid": "9cc7386a-3560-4682-83bd-83c21235cc5a", "collapsed": true, "_uuid": "84403d5b57db00917876269fd2962e25c17fe782"}, "cell_type": "code", "source": ["bar = tqdm_notebook(total=len(file_pointers))\n", "for i in range(len(file_pointers) - 1):\n", "    with open('train_example.bson', 'rb') as fbson:\n", "        fbson.seek(file_pointers[i])\n", "        bytes_chunk = fbson.read(file_pointers[i + 1] - file_pointers[i])\n", "        # Do something with bytes_chunk, for example: write to file, upload to Amazon S3, etc."], "outputs": [], "execution_count": null}], "nbformat_minor": 1}