{"cells":[{"metadata":{"_uuid":"b6c8ac12476d5331f2d156f77547fc9c06c24514","_cell_guid":"7c6a027b-7d9c-4071-bfb7-c68de7d20557","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1d9091dfa96462c7275aa8d682aea12d5023e69","collapsed":true,"_cell_guid":"00fb02a6-f1c3-4659-b268-c80214937353"},"cell_type":"markdown","source":"The bson files for this competition contain a list of dictionaries, one dictionary per product. Each dictionary contains a product id (key: _id), the category id of the product (key: category_id), and between 1-4 images, stored in a list (key: imgs). Each image list contains a single dictionary per image, which uses the format: {'picture': b'...binary string...'}. \n\nThe bson file can be read and processed iteratively. The following code shows how to read the data from the `train_example.bson` file."},{"metadata":{"_uuid":"00594caa126a1a133a7d40c3c462141ed3a66b23","_cell_guid":"d0ddf098-be4e-4b54-904f-758de23dd983","trusted":true},"cell_type":"code","source":"import io\nimport bson                       # this is installed with the pymongo package\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom skimage.io import imread\n# from skimage.data import imread   # or, whatever image library you prefer\nimport multiprocessing as mp      # will come in handy due to the size of the data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"642c4cd0a93320593e46f4834e416e9dab12126b","_cell_guid":"e839c025-3d76-43a2-948a-848baa8b2238","trusted":true},"cell_type":"code","source":"# Simple data processing\n\ndata = bson.decode_file_iter(open('../input/train_example.bson', 'rb'))\n\n# prod_to_category = dict()\n# prod_to_img = dict()\n\nprod_id = []\ncat_id = []\nimg_arr = []\n\nfor c, d in enumerate(data):\n    product_id = d['_id']\n    category_id = d['category_id'] # This won't be in Test data\n#     prod_to_category[product_id] = category_id\n    for e, pic in enumerate(d['imgs']):\n        picture = imread(io.BytesIO(pic['picture']))\n        \n        prod_id.append(product_id)\n        cat_id.append(category_id)\n        img_arr.append(picture)\n        \ndf = pd.DataFrame(list(zip(img_arr, prod_id, cat_id)), columns=['img_arr', 'prod_id', 'cat_id'])\n\n# prod_to_category = pd.DataFrame.from_dict(prod_to_category, orient='index')\n# prod_to_category.index.name = '_id'\n# prod_to_category.rename(columns={0: 'category_id'}, inplace=True)\n\n# prod_to_img = pd.DataFrame.from_dict(prod_to_img, orient='index')\n# prod_to_img.index.name = '_id'\n# prod_to_img.rename(columns={0: 'img'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df['prod_id'] == 13]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"945826d2181ff81505df3565ca4828186806f4eb","_cell_guid":"bda20ba9-03f2-4684-bdcd-a51ac2ad6291","trusted":true},"cell_type":"code","source":"plt.imshow(df['img_arr'][0]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb889cd763087b53283a1d36ce2fb2e7c7caed64","_cell_guid":"1611ed53-c2ef-4a52-8933-6c1c75e856ff"},"cell_type":"markdown","source":"For more efficient use of your resources, you can use the `multiprocessing` module to read and process the bson file.\n\nInspiration for this code is from:  https://stackoverflow.com/questions/43078980/python-multiprocessing-with-generator Note this may be slower on a small file, due to the overhead setting up the workers, but will be significantly faster for the large files."},{"metadata":{"_uuid":"7834cb784d38e49a1eaff35291330a833eb8f166","_cell_guid":"d7a759c2-fd85-4a22-b0e3-6c114fe5e757","trusted":true},"cell_type":"code","source":"NCORE =  8\n\nprod_id = mp.Manager().list()\ncat_id = mp.Manager().list()\nimg_arr = mp.Manager().list()\n\ndef process(q, iolock):\n    while True:\n        d = q.get()\n        if d is None:\n            break\n        product_id = d['_id']\n        category_id = d['category_id']\n        for e, pic in enumerate(d['imgs']):\n            picture = imread(io.BytesIO(pic['picture']))\n            # do something with the picture, etc\n            prod_id.append(product_id)\n            cat_id.append(category_id)\n            img_arr.append(picture)\n        \n    \nq = mp.Queue(maxsize=NCORE)\niolock = mp.Lock()\npool = mp.Pool(NCORE, initializer=process, initargs=(q, iolock))\n\n# process the file\n\ndata = bson.decode_file_iter(open('../input/train_example.bson', 'rb'))\nfor c, d in enumerate(data):\n    q.put(d)  # blocks until q below its max size\n\n# tell workers we're done\n\nfor _ in range(NCORE):  \n    q.put(None)\npool.close()\npool.join()\n\nprod_id = list(prod_id)\ncat_id = list(cat_id)\nimg_arr = list(img_arr)\n\ndf = pd.DataFrame(list(zip(img_arr, prod_id, cat_id)), columns=['img_arr', 'prod_id', 'cat_id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d287eb7f5e994a0afe55fb84e9bd2608939a0db","_cell_guid":"e5a8052d-e91f-4b0b-909e-a3f43865c2ef","trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6c136176a412b34a278162190e52422ec8a9df1","_cell_guid":"c12084c3-9d88-4802-819a-6382aef58093","trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['cat_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}