{"metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"mimetype": "text/x-python", "pygments_lexer": "ipython3", "version": "3.6.1", "file_extension": ".py", "name": "python", "nbconvert_exporter": "python", "codemirror_mode": {"name": "ipython", "version": 3}}}, "nbformat": 4, "nbformat_minor": 1, "cells": [{"cell_type": "markdown", "metadata": {}, "source": ["# Random item access from BSON file\n", "\n", "Here is one of some other methods to access item without iterating through the whole BSON file\n", "Idea is to store offsets and lenghts of all items and seek/read from binary file.\n", "\n", "Following code creates a dictionary with key indexing item `_id` and values `(offset, length)`. It takes around 3 mins to execute.\n"]}, {"outputs": [], "cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["import os\n", "import sys\n", "import numpy as np # linear algebra\n", "import pandas as pd\n", "import bson\n", "import cv2\n", "import matplotlib.pyplot as plt"]}, {"outputs": [], "cell_type": "code", "metadata": {"_cell_guid": "cfae6b73-63cf-4996-a306-ec3569562797", "_uuid": "07ae8272eeeae2e7bb5e9df164724721ba9c9225"}, "execution_count": null, "source": ["INPUT_PATH = os.path.join('..', 'input')\n", "\n", "import struct\n", "from tqdm import tqdm_notebook\n", "\n", "num_dicts = 7069896 # according to data page\n", "\n", "IDS_MAPPING = {}\n", "\n", "length_size = 4 # number of bytes decoding item length\n", "\n", "with open(os.path.join(INPUT_PATH, 'train.bson'), 'rb') as f, tqdm_notebook(total=num_dicts) as bar:\n", "    item_data = []\n", "    offset = 0\n", "    while True:        \n", "        bar.update()\n", "        f.seek(offset)\n", "        \n", "        item_length_bytes = f.read(length_size)     \n", "        if len(item_length_bytes) == 0:\n", "            break                \n", "        # Decode item length:\n", "        length = struct.unpack(\"<i\", item_length_bytes)[0]\n", "\n", "        f.seek(offset)\n", "        item_data = f.read(length)\n", "        assert len(item_data) == length, \"%i vs %i\" % (len(item_data), length)\n", "        \n", "        # Check if we can decode\n", "        item = bson.BSON.decode(item_data)\n", "        \n", "        IDS_MAPPING[item['_id']] = (offset, length)        \n", "        offset += length            \n", "            \n", "def get_item(item_id):\n", "    assert item_id in IDS_MAPPING\n", "    with open(os.path.join(INPUT_PATH, 'train.bson'), 'rb') as f:\n", "        offset, length = IDS_MAPPING[item_id]\n", "        f.seek(offset)\n", "        item_data = f.read(length)\n", "        return bson.BSON.decode(item_data)"]}, {"outputs": [], "cell_type": "code", "metadata": {}, "execution_count": null, "source": ["CATEGORY_NAMES_DF = pd.read_csv(os.path.join(INPUT_PATH, 'category_names.csv'))\n", "level_tags = CATEGORY_NAMES_DF.columns[1:]\n", "\n", "\n", "def decode(data):\n", "    arr = np.asarray(bytearray(data), dtype=np.uint8)\n", "    img = cv2.imdecode(arr, cv2.IMREAD_COLOR)\n", "    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB) \n", "\n", "# Method to compose a single image from 1 - 4 images\n", "def decode_images(item_imgs):\n", "    nx = 2 if len(item_imgs) > 1 else 1\n", "    ny = 2 if len(item_imgs) > 2 else 1\n", "    composed_img = np.zeros((ny * 180, nx * 180, 3), dtype=np.uint8)\n", "    for i, img_dict in enumerate(item_imgs):\n", "        img = decode(img_dict['picture'])\n", "        h, w, _ = img.shape        \n", "        xstart = (i % nx) * 180\n", "        xend = xstart + w\n", "        ystart = (i // nx) * 180\n", "        yend = ystart + h\n", "        composed_img[ystart:yend, xstart:xend] = img\n", "    return composed_img\n", "\n", "item = get_item(1234)\n", "\n", "mask = CATEGORY_NAMES_DF['category_id'] == item['category_id']    \n", "cat_levels = CATEGORY_NAMES_DF[mask][level_tags].values.tolist()[0]\n", "cat_levels = [c[:25] for c in cat_levels]\n", "title = str(item['category_id']) + '\\n'\n", "title += '\\n'.join(cat_levels)\n", "plt.title(title)\n", "plt.imshow(decode_images(item['imgs']))\n", "_ = plt.axis('off')"]}, {"outputs": [], "cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": []}]}