{"cells": [{"cell_type": "markdown", "metadata": {"_cell_guid": "e3e4a214-a599-4588-a2d6-9c45810bc666", "_uuid": "d491c01dafa9bf016d93acbfc9f7aa665943c2b9"}, "source": ["Original work by https://www.kaggle.com/inversion/processing-bson-files\n", "Add product_to_category in the definition of process for the multiprocessing, so it works for the Win10.  "]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "774b714d-1ff4-40bf-af9e-af5c3f449245", "_uuid": "9e92ec69cb9fc972d71081dfc34b7fdd360bb7b9", "collapsed": true}, "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "import io\n", "import os\n", "import bson\n", "import matplotlib.pyplot as plt\n", "from skimage.data import imread \n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "a99d5c77-e837-4804-a451-034ab96790eb", "_uuid": "beeafda7ae204fb786674d3071c38ceb2a6e0e94", "collapsed": true}, "source": ["data_dir='../input'\n", "data = bson.decode_file_iter(open(os.path.join(data_dir,'train_example.bson'),'rb'))"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "2f33a3d7-f6da-4142-a630-b9161fac3f59", "_uuid": "5f9a3a9888b47ef67b2b7604c4d4ea460aa3bb64", "collapsed": true}, "source": ["product_to_category=dict()\n", "for c,d in enumerate(data):\n", "    product_id = d['_id']\n", "    category_id = d['category_id']\n", "    product_to_category[product_id] = category_id\n", "    for e,pic in enumerate(d['imgs']):\n", "        picture = imread(io.BytesIO(pic['picture']))"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "ffed2f33-0565-41e8-8873-f78e508039ce", "_uuid": "5c1ab48ff9bd0e64ca31699f0172e7dcf8b1536c", "collapsed": true}, "source": ["product_to_category = pd.DataFrame.from_dict(product_to_category,orient='index')"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "b90c697b-8157-423c-9ac2-26071fe7a724", "_uuid": "633b1e4f8af81b53d0d8e53c9fa26b6cfcd9741b", "collapsed": true}, "source": ["product_to_category.index.name='_id'\n", "product_to_category.rename(columns={0:'category_id'},inplace=True)"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "9394485c-45a2-440e-bf50-d7f47a5c58c4", "_uuid": "9bd254de5dcc6e85dd874a82e6a835d978edd71c", "collapsed": true}, "source": ["product_to_category.head()"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "f3a1431c-5378-4558-bf24-8df83243bce7", "_uuid": "898e1ca95099cd5e290b2f71b74f492e0d75fae0", "collapsed": true}, "source": ["import multiprocessing as mp"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "06f39314-91f5-4507-86cb-ded78d77ce91", "_uuid": "779d8522b8d48dd6360163796d118f03ad0d4f02", "collapsed": true}, "source": ["NCORE =  2\n", "manager = mp.Manager()\n", "\n"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "abdd5c3e-e74d-464d-abfc-332daf697f56", "_uuid": "4695124e94140adeba88c7390c4036188de9d7b1", "collapsed": true}, "source": ["product_to_category = manager.dict()"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "a8df255e-6872-4d79-8e69-ac0d65281f95", "_uuid": "7ed0df94de0abebe76bd70c4473ca8fa6b8e7238", "collapsed": true}, "source": ["def process(q, iolock, product_to_category):\n", "    while True:\n", "        d=q.get()\n", "        if d is None:\n", "            break\n", "        product_id = d['_id']\n", "        category_id = d['category_id']\n", "        product_to_category[product_id] = category_id\n", "        for e,pic in enumerate(d['imgs']):\n", "            picture = imread(io.BytesIO(pic['picture']))\n", "            "]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "eaa03f38-95aa-47c2-a4ef-fba2552e66bb", "_uuid": "b230ef2ab2c42f72b72e2cfd3e79d8b855dc3f67", "collapsed": true}, "source": ["data = bson.decode_file_iter(open(os.path.join(data_dir,'train_example.bson'),'rb'))\n", "q=mp.Queue(maxsize=NCORE)\n", "iolock=mp.Lock()\n", "pool = mp.Pool(NCORE, initializer=process, initargs=(q, iolock, product_to_category))\n", "for c,d in enumerate(data):\n", "    q.put(d)\n", "    \n", "for _ in range(NCORE):\n", "    q.put(None)\n", "pool.close()\n", "pool.join()"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "c2f752db-08cf-477d-825d-9fab0da96f1f", "_uuid": "e56e5858f204dca2ad532377ea95c54ac0535e1b", "collapsed": true}, "source": ["product_to_category=dict(product_to_category)"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "75194f81-689f-4d02-b310-7bfeb519a613", "_uuid": "c86881d1bd106d860d8a182bafad03c09f94c4cc", "collapsed": true}, "source": ["product_to_category"]}, {"cell_type": "code", "execution_count": null, "outputs": [], "metadata": {"_cell_guid": "ec24bcfb-3c55-442f-810a-a85f77df9eae", "_uuid": "d8df64d81c51c86f964a9357d79dd2b2264e81fb", "collapsed": true}, "source": []}], "nbformat_minor": 1, "metadata": {"language_info": {"mimetype": "text/x-python", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.3"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "nbformat": 4}