{"metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}, "language_info": {"nbconvert_exporter": "python", "version": "3.6.3", "name": "python", "pygments_lexer": "ipython3", "codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python"}}, "cells": [{"source": ["If you are like me and have limited space on your laptop, this notebook will help you alot. I wanted to get a larger training file that I could use for building models on my laptop and then testing it on that small subset and move it to the main training on AWS. Besides, I have a limited internet connection. In this notebook, I'm generating a larger training sample than the one provided. Allowing me to build and test algorithms before moving it to larger AWS machine to excute on the entire dataset. Through testing Kaggle file limits, I found that 40MB is a good size acceptable by the Notebook environment here on Kaggle and 5MB for test. I wish I was able to make it about 1GB for train and about 100MB for testing. That would have been a better."], "metadata": {"_cell_guid": "420063d3-cf5a-4452-9dec-38947cd46869", "_uuid": "1a7ec17e56a9f4a65167de0b0e3173c1d9716f21"}, "cell_type": "markdown"}, {"source": ["import bson \n", "from bson import BSON\n", "import os\n", "from tqdm import tqdm_notebook as tqdm\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"..\"]).decode(\"utf8\"))"], "metadata": {"_cell_guid": "6108af34-442b-4265-8181-774716a086a0", "collapsed": true, "_uuid": "59f7b7c6acf4afa96af4f156dff403acb2602b75"}, "execution_count": null, "outputs": [], "cell_type": "code"}, {"source": ["if os.path.isfile(\"train_small.bson\"):\n", "    os.remove(\"train_small.bson\")\n", "\n", "if os.path.isfile(\"test_small.bson\"):\n", "    os.remove(\"test_small.bson\")\n", "    \n", "f = open('train_small.bson', 'ab+')\n", "\n", "MAX_TRAIN_FILE_SIZE = 40 * 1024 * 1024\n", "MAX_TEST_FILE_SIZE = 5 * 1024 * 1024\n", "\n", "data = bson.decode_file_iter(open('../input/train.bson', 'rb'))\n", "cat_list = list()\n", "pbar = tqdm(total=MAX_TRAIN_FILE_SIZE)\n", "sz_last = 0\n", "for d in data:\n", "    category_id = d['category_id']\n", "    if not category_id in cat_list:\n", "        f.write(BSON.encode(d))\n", "        cat_list.append(category_id)        \n", "        fsz = f.tell()\n", "        sz_diff = fsz - sz_last\n", "        pbar.update(sz_diff)\n", "        sz_last = fsz\n", "        if fsz > MAX_TRAIN_FILE_SIZE:\n", "            break\n", "pbar.close()\n", "f.close()\n", "print(\"Saved {:d} categories in train.\".format(len(cat_list)))\n", "\n", "## Saving test sample as well.\n", "f = open('test_small.bson', 'ab+')\n", "data = bson.decode_file_iter(open('../input/test.bson', 'rb'))\n", "pbar = tqdm(total=MAX_TEST_FILE_SIZE)\n", "sz_last = 0\n", "num_samples = 0\n", "for d in data:\n", "    f.write(BSON.encode(d))\n", "    num_samples += 1\n", "    fsz = f.tell()\n", "    sz_diff = fsz - sz_last\n", "    pbar.update(sz_diff)\n", "    sz_last = fsz\n", "    if fsz > MAX_TEST_FILE_SIZE:\n", "        break\n", "pbar.close()\n", "f.close()\n", "print(\"Saved {:d} samples in test.\".format(num_samples))"], "metadata": {"_cell_guid": "e2a8b026-f27d-45b1-8f55-8cdd7f6c1aa3", "collapsed": true, "_uuid": "d61a6778ed4c697e6c55780ce9d8fcb64198c963"}, "execution_count": null, "outputs": [], "cell_type": "code"}, {"source": [], "metadata": {"_cell_guid": "f0cb7be4-8fbb-46f0-ad39-87b5a084ed34", "collapsed": true, "_uuid": "1d63ca0590af1a2a071370e161debd9d1a7b4fdd"}, "execution_count": null, "outputs": [], "cell_type": "code"}], "nbformat_minor": 1, "nbformat": 4}