{"metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "version": "3.6.1", "codemirror_mode": {"name": "ipython", "version": 3}}}, "nbformat": 4, "nbformat_minor": 1, "cells": [{"cell_type": "code", "metadata": {"_uuid": "b6c8ac12476d5331f2d156f77547fc9c06c24514", "_cell_guid": "7c6a027b-7d9c-4071-bfb7-c68de7d20557"}, "execution_count": null, "outputs": [], "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."]}, {"cell_type": "markdown", "metadata": {"_uuid": "e1d9091dfa96462c7275aa8d682aea12d5023e69", "collapsed": true, "_cell_guid": "00fb02a6-f1c3-4659-b268-c80214937353"}, "source": ["The bson files for this competition contain a list of dictionaries, one dictionary per product. Each dictionary contains a product id (key: _id), the category id of the product (key: category_id), and between 1-4 images, stored in a list (key: imgs). Each image list contains a single dictionary per image, which uses the format: {'picture': b'...binary string...'}. \n", "\n", "The bson file can be read and processed iteratively. The following code shows how to read the data from the `train_example.bson` file."]}, {"cell_type": "code", "metadata": {"_uuid": "00594caa126a1a133a7d40c3c462141ed3a66b23", "collapsed": true, "_cell_guid": "d0ddf098-be4e-4b54-904f-758de23dd983"}, "execution_count": null, "outputs": [], "source": ["import io\n", "import bson                       # this is installed with the pymongo package\n", "import matplotlib.pyplot as plt\n", "from skimage.data import imread   # or, whatever image library you prefer\n", "import multiprocessing as mp      # will come in handy due to the size of the data"]}, {"cell_type": "code", "metadata": {"_uuid": "642c4cd0a93320593e46f4834e416e9dab12126b", "collapsed": true, "_cell_guid": "e839c025-3d76-43a2-948a-848baa8b2238"}, "execution_count": null, "outputs": [], "source": ["# Simple data processing\n", "\n", "data = bson.decode_file_iter(open('../input/train_example.bson', 'rb'))\n", "\n", "prod_to_category = dict()\n", "\n", "for c, d in enumerate(data):\n", "    product_id = d['_id']\n", "    category_id = d['category_id'] # This won't be in Test data\n", "    prod_to_category[product_id] = category_id\n", "    for e, pic in enumerate(d['imgs']):\n", "        picture = imread(io.BytesIO(pic['picture']))\n", "        # do something with the picture, etc\n", "\n", "prod_to_category = pd.DataFrame.from_dict(prod_to_category, orient='index')\n", "prod_to_category.index.name = '_id'\n", "prod_to_category.rename(columns={0: 'category_id'}, inplace=True)"]}, {"cell_type": "code", "metadata": {"_uuid": "f9db8d72a4248f449de9b3264242ea09b1320a64", "collapsed": true, "_cell_guid": "cfd09357-c40d-4e86-a7c9-a0e75face1cf"}, "execution_count": null, "outputs": [], "source": ["prod_to_category.head()"]}, {"cell_type": "code", "metadata": {"_uuid": "945826d2181ff81505df3565ca4828186806f4eb", "collapsed": true, "_cell_guid": "bda20ba9-03f2-4684-bdcd-a51ac2ad6291"}, "execution_count": null, "outputs": [], "source": ["plt.imshow(picture);"]}, {"cell_type": "markdown", "metadata": {"_uuid": "bb889cd763087b53283a1d36ce2fb2e7c7caed64", "_cell_guid": "1611ed53-c2ef-4a52-8933-6c1c75e856ff"}, "source": ["For more efficient use of your resources, you can use the `multiprocessing` module to read and process the bson file.\n", "\n", "Inspiration for this code is from:  https://stackoverflow.com/questions/43078980/python-multiprocessing-with-generator Note this may be slower on a small file, due to the overhead setting up the workers, but will be significantly faster for the large files."]}, {"cell_type": "code", "metadata": {"_uuid": "7834cb784d38e49a1eaff35291330a833eb8f166", "collapsed": true, "_cell_guid": "d7a759c2-fd85-4a22-b0e3-6c114fe5e757"}, "execution_count": null, "outputs": [], "source": ["NCORE =  8\n", "\n", "prod_to_category = mp.Manager().dict() # note the difference\n", "\n", "def process(q, iolock):\n", "    while True:\n", "        d = q.get()\n", "        if d is None:\n", "            break\n", "        product_id = d['_id']\n", "        category_id = d['category_id']\n", "        prod_to_category[product_id] = category_id\n", "        for e, pic in enumerate(d['imgs']):\n", "            picture = imread(io.BytesIO(pic['picture']))\n", "            # do something with the picture, etc\n", "    \n", "q = mp.Queue(maxsize=NCORE)\n", "iolock = mp.Lock()\n", "pool = mp.Pool(NCORE, initializer=process, initargs=(q, iolock))\n", "\n", "# process the file\n", "\n", "data = bson.decode_file_iter(open('../input/train_example.bson', 'rb'))\n", "for c, d in enumerate(data):\n", "    q.put(d)  # blocks until q below its max size\n", "\n", "# tell workers we're done\n", "\n", "for _ in range(NCORE):  \n", "    q.put(None)\n", "pool.close()\n", "pool.join()\n", "\n", "# convert back to normal dictionary\n", "prod_to_category = dict(prod_to_category)\n", "\n", "prod_to_category = pd.DataFrame.from_dict(prod_to_category, orient='index')\n", "prod_to_category.index.name = '_id'\n", "prod_to_category.rename(columns={0: 'category_id'}, inplace=True)"]}, {"cell_type": "code", "metadata": {"_uuid": "6d287eb7f5e994a0afe55fb84e9bd2608939a0db", "collapsed": true, "_cell_guid": "e5a8052d-e91f-4b0b-909e-a3f43865c2ef"}, "execution_count": null, "outputs": [], "source": ["prod_to_category.head()"]}, {"cell_type": "code", "metadata": {"_uuid": "a6c136176a412b34a278162190e52422ec8a9df1", "collapsed": true, "_cell_guid": "c12084c3-9d88-4802-819a-6382aef58093"}, "execution_count": null, "outputs": [], "source": []}]}