{"nbformat":4,"cells":[{"source":"This notebook contains a generator class for Keras called `BSONIterator` that can read directly from the BSON data. You can use it in combination with `ImageDataGenerator` for doing data augmentation.","metadata":{"_cell_guid":"75d61c38-954c-4bd1-a127-5d17e1812402","_uuid":"2ae1f395d8768512da0739dded661e329dbfa14e"},"cell_type":"markdown"},{"source":"import os, sys, math, io\nimport numpy as np\nimport pandas as pd\nimport multiprocessing as mp\nimport bson\nimport struct\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport keras\nfrom keras.preprocessing.image import load_img, img_to_array\nimport tensorflow as tf\n\nfrom collections import defaultdict\nfrom tqdm import *\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","outputs":[],"metadata":{"_cell_guid":"4036f11e-5223-46e5-8948-5cbf70a9bf73","_uuid":"3767d2738682e30c292a466f66bc75fcc80a5076"},"execution_count":null,"cell_type":"code"},{"source":"keras.__version__, tf.__version__","outputs":[],"metadata":{"_cell_guid":"3601ddc2-5dca-463c-be54-a297e019469a","_uuid":"41fb122f53b29ff6c7cdba4f08001fa8204fab34"},"execution_count":null,"cell_type":"code"},{"source":"data_dir = \"../input/\"\n\ntrain_bson_path = os.path.join(data_dir, \"train.bson\")\nnum_train_products = 7069896\n\n# train_bson_path = os.path.join(data_dir, \"train_example.bson\")\n# num_train_products = 82\n\ntest_bson_path = os.path.join(data_dir, \"test.bson\")\nnum_test_products = 1768182","outputs":[],"metadata":{"_cell_guid":"dfec6cb3-3b14-44cc-920c-c4bba4f79589","collapsed":true,"_uuid":"1d92f3f86af7906e204eec1d536bd29878fed02d"},"execution_count":null,"cell_type":"code"},{"source":"# Part 1: Create lookup tables\n\nThe generator uses several lookup tables that describe the layout of the BSON file, which products and images are part of the training/validation sets, and so on.\n\nYou only need to generate these tables once, as they get saved to CSV files. If you already have these CSV files, skip to part 2.","metadata":{"_cell_guid":"3c24b66f-0f55-4007-86d9-aae6ec1c16bc","_uuid":"0c1457925b72937cc9b8d431142f06098cbfe06c"},"cell_type":"markdown"},{"source":"## Lookup table for categories","metadata":{"_cell_guid":"593c6f49-83eb-4491-bcd1-0131845e395c","_uuid":"7ea59756f0ab3eb271e2b5d6495e6a158311de35"},"cell_type":"markdown"},{"source":"categories_path = os.path.join(data_dir, \"category_names.csv\")\ncategories_df = pd.read_csv(categories_path, index_col=\"category_id\")\n\n# Maps the category_id to an integer index. This is what we'll use to\n# one-hot encode the labels.\ncategories_df[\"category_idx\"] = pd.Series(range(len(categories_df)), index=categories_df.index)\n\ncategories_df.to_csv(\"categories.csv\")\ncategories_df.head()","outputs":[],"metadata":{"_cell_guid":"c049b397-8b31-4389-9657-f0281c5f7b4f","_uuid":"c2a19dc1ea89274a1ab3332fa635e6dfdc385ce6"},"execution_count":null,"cell_type":"code"},{"source":"Create dictionaries for quick lookup of `category_id` to `category_idx` mapping.","metadata":{"_cell_guid":"92491e45-56da-4d64-a887-6cf7a8c8cd30","_uuid":"05ccbed26b3518d90ddcc9c26668f6008b89cb0f"},"cell_type":"markdown"},{"source":"def make_category_tables():\n    cat2idx = {}\n    idx2cat = {}\n    for ir in categories_df.itertuples():\n        category_id = ir[0]\n        category_idx = ir[4]\n        cat2idx[category_id] = category_idx\n        idx2cat[category_idx] = category_id\n    return cat2idx, idx2cat","outputs":[],"metadata":{"_cell_guid":"7aad702e-68b9-4c94-a6e7-910283629011","collapsed":true,"_uuid":"1c40f20530ad22246d6941845c89b210b4a85368"},"execution_count":null,"cell_type":"code"},{"source":"cat2idx, idx2cat = make_category_tables()","outputs":[],"metadata":{"_cell_guid":"99c21338-72c9-406a-bc3a-ba33d7310086","collapsed":true,"_uuid":"b68af0dda86d6c6af127a5bf634778fb63bda958"},"execution_count":null,"cell_type":"code"},{"source":"# Test if it works:\ncat2idx[1000012755], idx2cat[4]","outputs":[],"metadata":{"_cell_guid":"530dac1b-2891-42ff-ab7f-ea1fe8ef01e6","_uuid":"3fff32a945788e89e676c68fc20ff138ac72f754"},"execution_count":null,"cell_type":"code"},{"source":"## Read the BSON files\n\nWe store the offsets and lengths of all items, allowing us random access to the items later.\n\nInspired by code from: https://www.kaggle.com/vfdev5/random-item-access\n\nNote: this takes a few minutes to execute, but we only have to do it once (we'll save the table to a CSV file afterwards).","metadata":{"_cell_guid":"d4518fba-7f26-45c1-ae3f-daf72ffbbee9","_uuid":"51f82526d1ce6b7ab0b4d5b370f0f2f9601dbc66"},"cell_type":"markdown"},{"source":"def read_bson(bson_path, num_records, with_categories):\n    rows = {}\n    with open(bson_path, \"rb\") as f, tqdm(total=num_records) as pbar:\n        offset = 0\n        while True:\n            item_length_bytes = f.read(4)\n            if len(item_length_bytes) == 0:\n                break\n\n            length = struct.unpack(\"<i\", item_length_bytes)[0]\n\n            f.seek(offset)\n            item_data = f.read(length)\n            assert len(item_data) == length\n\n            item = bson.BSON.decode(item_data)\n            product_id = item[\"_id\"]\n            num_imgs = len(item[\"imgs\"])\n\n            row = [num_imgs, offset, length]\n            if with_categories:\n                row += [item[\"category_id\"]]\n            rows[product_id] = row\n\n            offset += length\n            f.seek(offset)\n            pbar.update()\n\n    columns = [\"num_imgs\", \"offset\", \"length\"]\n    if with_categories:\n        columns += [\"category_id\"]\n\n    df = pd.DataFrame.from_dict(rows, orient=\"index\")\n    df.index.name = \"product_id\"\n    df.columns = columns\n    df.sort_index(inplace=True)\n    return df","outputs":[],"metadata":{"_cell_guid":"2419b6a8-d141-4f5d-a868-778e0483fc40","collapsed":true,"_uuid":"50f1fa1bc0a9c0597ce90d7c7daf2d0b7a788467"},"execution_count":null,"cell_type":"code"},{"source":"%time train_offsets_df = read_bson(train_bson_path, num_records=num_train_products, with_categories=True)","outputs":[],"metadata":{"_cell_guid":"54c03644-11a0-425b-bf2e-b412f087a390","_uuid":"3607b538f418145677d61bc80f6a0b6e97bd1c95"},"execution_count":null,"cell_type":"code"},{"source":"train_offsets_df.head()","outputs":[],"metadata":{"_cell_guid":"4e6d4a2e-2014-4610-b624-0a5de1ae803e","_uuid":"507ae4e5fceec73fdfb08494e0dcde86b0d62a29"},"execution_count":null,"cell_type":"code"},{"source":"train_offsets_df.to_csv(\"train_offsets.csv\")","outputs":[],"metadata":{"_cell_guid":"8d24b1ed-8f75-4e29-a70d-627ae11deb45","collapsed":true,"_uuid":"3e5daf1253397d05065568ad3ce69be73042cc59"},"execution_count":null,"cell_type":"code"},{"source":"# How many products?\nlen(train_offsets_df)","outputs":[],"metadata":{"_cell_guid":"0c331f10-e2af-43ee-8bcd-1b4058430cbd","_uuid":"254a22b36211385bc943f1c3d10914f75548b5e0"},"execution_count":null,"cell_type":"code"},{"source":"# How many categories?\nlen(train_offsets_df[\"category_id\"].unique())","outputs":[],"metadata":{"_cell_guid":"c357322f-5026-4454-9c0f-a2d40b72c99a","_uuid":"9aa11d00937321bcfe68f80be0cc17a2cab39e65"},"execution_count":null,"cell_type":"code"},{"source":"# How many images in total?\ntrain_offsets_df[\"num_imgs\"].sum()","outputs":[],"metadata":{"_cell_guid":"7eac8e95-7930-4e07-a869-5fcd0eb8e1a1","_uuid":"f1ed766c559b2b7848e452b0b94421c3d4767c81"},"execution_count":null,"cell_type":"code"},{"source":"## Create a random train/validation split\n\nWe split on products, not on individual images. Since some of the categories only have a few products, we do the split separately for each category.\n\nThis creates two new tables, one for the training images and one for the validation images. There is a row for every single image, so if a product has more than one image it occurs more than once in the table.","metadata":{"_cell_guid":"663b73b1-4de8-4adc-98b8-475403bd752d","_uuid":"39e87235e9b8416ccbf6ea0909be65cfe6d00b65"},"cell_type":"markdown"},{"source":"def make_val_set(df, split_percentage=0.2, drop_percentage=0.):\n    # Find the product_ids for each category.\n    category_dict = defaultdict(list)\n    for ir in tqdm(df.itertuples()):\n        category_dict[ir[4]].append(ir[0])\n\n    train_list = []\n    val_list = []\n    with tqdm(total=len(df)) as pbar:\n        for category_id, product_ids in category_dict.items():\n            category_idx = cat2idx[category_id]\n\n            # Randomly remove products to make the dataset smaller.\n            keep_size = int(len(product_ids) * (1. - drop_percentage))\n            if keep_size < len(product_ids):\n                product_ids = np.random.choice(product_ids, keep_size, replace=False)\n\n            # Randomly choose the products that become part of the validation set.\n            val_size = int(len(product_ids) * split_percentage)\n            if val_size > 0:\n                val_ids = np.random.choice(product_ids, val_size, replace=False)\n            else:\n                val_ids = []\n\n            # Create a new row for each image.\n            for product_id in product_ids:\n                row = [product_id, category_idx]\n                for img_idx in range(df.loc[product_id, \"num_imgs\"]):\n                    if product_id in val_ids:\n                        val_list.append(row + [img_idx])\n                    else:\n                        train_list.append(row + [img_idx])\n                pbar.update()\n                \n    columns = [\"product_id\", \"category_idx\", \"img_idx\"]\n    train_df = pd.DataFrame(train_list, columns=columns)\n    val_df = pd.DataFrame(val_list, columns=columns)   \n    return train_df, val_df","outputs":[],"metadata":{"_cell_guid":"24e8db94-3a2b-453d-a8ac-1de2460ed129","collapsed":true,"_uuid":"4d702f77af77901c7ed547eebb1da9146afea667"},"execution_count":null,"cell_type":"code"},{"source":"Create a 80/20 split. Also drop 90% of all products to make the dataset more manageable. (Note: if `drop_percentage` > 0, the progress bar doesn't go all the way.)","metadata":{"_cell_guid":"8df853cd-b2e5-4c05-8162-e332c5995bdc","_uuid":"6e3c437a8169a31e9e4f4170f70d48668f8f0934"},"cell_type":"markdown"},{"source":"train_images_df, val_images_df = make_val_set(train_offsets_df, split_percentage=0.2, \n                                              drop_percentage=0.9)","outputs":[],"metadata":{"_cell_guid":"a27d017f-8ac2-4480-8b48-7d092be8de82","_uuid":"586a0ad09c6ce8af090277d0ac930ba53eb3a8d3"},"execution_count":null,"cell_type":"code"},{"source":"train_images_df.head()","outputs":[],"metadata":{"_cell_guid":"92517b1e-fa81-4d89-ba41-b6a8614937dd","_uuid":"3016b0855ca99e3f39826e5c29eeae80fec42b62"},"execution_count":null,"cell_type":"code"},{"source":"val_images_df.head()","outputs":[],"metadata":{"_cell_guid":"c3e170c3-4f7a-499a-8bca-22f5e6a20693","_uuid":"8b7255493416498f9151645648c560744187da03"},"execution_count":null,"cell_type":"code"},{"source":"print(\"Number of training images:\", len(train_images_df))\nprint(\"Number of validation images:\", len(val_images_df))\nprint(\"Total images:\", len(train_images_df) + len(val_images_df))","outputs":[],"metadata":{"_cell_guid":"0cb17194-01c4-4b38-a140-ec8330e7aaa8","_uuid":"5b9ee034409bed1eeafe54bceb3c008c4d4c9334"},"execution_count":null,"cell_type":"code"},{"source":"Are all categories represented in the train/val split? (Note: if the drop percentage is high, then very small categories won't have enough products left to make it into the validation set.)","metadata":{"_cell_guid":"f0b89d28-b068-44d2-909d-d43e0dfe1ab2","_uuid":"7aaae46c2b9b773cfee6430e4049c7abe0b69af9"},"cell_type":"markdown"},{"source":"len(train_images_df[\"category_idx\"].unique()), len(val_images_df[\"category_idx\"].unique())","outputs":[],"metadata":{"_cell_guid":"9685e057-5606-4e73-9ab1-33e2d638faa1","_uuid":"5abbe4cf931245641be378d677963dde40800cb7"},"execution_count":null,"cell_type":"code"},{"source":"Quickly verify that the split really is approximately 80-20:","metadata":{"_cell_guid":"59abf374-4259-4e2b-8bcb-f75947f9d4dc","_uuid":"8090c3dfdf328ea0e9f89b2280126a5a58840c09"},"cell_type":"markdown"},{"source":"category_idx = 619\nnum_train = np.sum(train_images_df[\"category_idx\"] == category_idx)\nnum_val = np.sum(val_images_df[\"category_idx\"] == category_idx)\nnum_val / num_train","outputs":[],"metadata":{"_cell_guid":"5666aad4-577d-431e-a26c-39b289abc3da","_uuid":"1bdbdfe266a8c4510a51f89896aa3c9ad17d9f72"},"execution_count":null,"cell_type":"code"},{"source":"Close enough. ;-) Remember that we split on products but not all products have the same number of images, which is where the slightly discrepancy comes from. (Also, there tend to be fewer validation images if `drop_percentage` > 0.)","metadata":{"_cell_guid":"624c622f-059a-4ce0-b68e-f373dba30b03","_uuid":"1bb825a934deaa8a287a0243927193377e6fa433"},"cell_type":"markdown"},{"source":"Save the lookup tables as CSV so that we don't need to repeat the above procedure again.","metadata":{"_cell_guid":"f84f672e-c16d-47a7-9d1c-84f9a0fb3e22","_uuid":"9450109fb5c910c9e7634855b7445fc4260097a7"},"cell_type":"markdown"},{"source":"train_images_df.to_csv(\"train_images.csv\")\nval_images_df.to_csv(\"val_images.csv\")","outputs":[],"metadata":{"_cell_guid":"c880c70f-dd98-4d61-b65f-f230abffdefd","collapsed":true,"_uuid":"747743164ea6dae14a5de4eb624465cb806089ab"},"execution_count":null,"cell_type":"code"},{"source":"# Part 2: The generator","metadata":{"_cell_guid":"4212f62a-86c3-4a13-8882-b4c90f64dac5","_uuid":"8a4bd65c90576d611340e36bce5f49896d942ed1"},"cell_type":"markdown"},{"source":"First load the lookup tables from the CSV files (you don't need to do this if you just did all the steps from part 1).","metadata":{"_cell_guid":"1da50467-d11d-417e-9367-bd8574dfe61a","_uuid":"55331a57195739250c5b530160cf32a57b9a2464"},"cell_type":"markdown"},{"source":"categories_df = pd.read_csv(\"categories.csv\", index_col=0)\ncat2idx, idx2cat = make_category_tables()\n\ntrain_offsets_df = pd.read_csv(\"train_offsets.csv\", index_col=0)\ntrain_images_df = pd.read_csv(\"train_images.csv\", index_col=0)\nval_images_df = pd.read_csv(\"val_images.csv\", index_col=0)","outputs":[],"metadata":{"_cell_guid":"19c24f98-551d-4b95-b8f7-8497ab09eb56","collapsed":true,"_uuid":"0c0dadb16f6b094748e967e9c682bb18b3b5c336"},"execution_count":null,"cell_type":"code"},{"source":"The Keras generator is implemented by the `BSONIterator` class. It creates batches of images (and their one-hot encoded labels) directly from the BSON file. It can be used with multiple workers.\n\n**Note:** For fastest results, put the train.bson and test.bson files on a fast drive (SSD).\n\nSee also the code in: https://github.com/fchollet/keras/blob/master/keras/preprocessing/image.py","metadata":{"_cell_guid":"17cb7bb4-4941-4d0d-8fa5-2dd60b7014ce","_uuid":"edbb272356d65e3ec9b540bd4597464ae4afa3c5"},"cell_type":"markdown"},{"source":"from keras.preprocessing.image import Iterator\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import backend as K\n\nclass BSONIterator(Iterator):\n    def __init__(self, bson_file, images_df, offsets_df, num_class,\n                 image_data_generator, lock, target_size=(180, 180), \n                 with_labels=True, batch_size=32, shuffle=False, seed=None):\n\n        self.file = bson_file\n        self.images_df = images_df\n        self.offsets_df = offsets_df\n        self.with_labels = with_labels\n        self.samples = len(images_df)\n        self.num_class = num_class\n        self.image_data_generator = image_data_generator\n        self.target_size = tuple(target_size)\n        self.image_shape = self.target_size + (3,)\n\n        print(\"Found %d images belonging to %d classes.\" % (self.samples, self.num_class))\n\n        super(BSONIterator, self).__init__(self.samples, batch_size, shuffle, seed)\n        self.lock = lock\n\n    def _get_batches_of_transformed_samples(self, index_array):\n        batch_x = np.zeros((len(index_array),) + self.image_shape, dtype=K.floatx())\n        if self.with_labels:\n            batch_y = np.zeros((len(batch_x), self.num_class), dtype=K.floatx())\n\n        for i, j in enumerate(index_array):\n            # Protect file and dataframe access with a lock.\n            with self.lock:\n                image_row = self.images_df.iloc[j]\n                product_id = image_row[\"product_id\"]\n                offset_row = self.offsets_df.loc[product_id]\n\n                # Read this product's data from the BSON file.\n                self.file.seek(offset_row[\"offset\"])\n                item_data = self.file.read(offset_row[\"length\"])\n\n            # Grab the image from the product.\n            item = bson.BSON.decode(item_data)\n            img_idx = image_row[\"img_idx\"]\n            bson_img = item[\"imgs\"][img_idx][\"picture\"]\n\n            # Load the image.\n            img = load_img(io.BytesIO(bson_img), target_size=self.target_size)\n\n            # Preprocess the image.\n            x = img_to_array(img)\n            x = self.image_data_generator.random_transform(x)\n            x = self.image_data_generator.standardize(x)\n\n            # Add the image and the label to the batch (one-hot encoded).\n            batch_x[i] = x\n            if self.with_labels:\n                batch_y[i, image_row[\"category_idx\"]] = 1\n\n        if self.with_labels:\n            return batch_x, batch_y\n        else:\n            return batch_x\n\n    def next(self):\n        with self.lock:\n            index_array = next(self.index_generator)\n        return self._get_batches_of_transformed_samples(index_array)","outputs":[],"metadata":{"_cell_guid":"78d8b565-336e-4836-b613-768bb6581499","collapsed":true,"_uuid":"c43d82c645b098b2dd8fc0962bd392bdfc43057d"},"execution_count":null,"cell_type":"code"},{"source":"train_bson_file = open(train_bson_path, \"rb\")","outputs":[],"metadata":{"_cell_guid":"86525843-13cc-413e-b372-085e4864a45a","collapsed":true,"_uuid":"a1de21a39914a95128adb7675b5fa17a5284f338"},"execution_count":null,"cell_type":"code"},{"source":"Because the training and validation generators read from the same BSON file, they need to use the same lock to protect it.","metadata":{"_cell_guid":"12b5adf3-fb74-4111-9075-442c466d8bb7","_uuid":"eb9dca40893f7ba7b44bf623f21ffd94889f5b6d"},"cell_type":"markdown"},{"source":"import threading\nlock = threading.Lock()","outputs":[],"metadata":{"_cell_guid":"182e1c6b-fc59-4244-9aed-7621ee5fbda2","collapsed":true,"_uuid":"fe904e99099d3f609e302e012421ac6cc230a959"},"execution_count":null,"cell_type":"code"},{"source":"Create a generator for training and a generator for validation.","metadata":{"_cell_guid":"2742b451-159b-4353-a7eb-96885fdf8919","_uuid":"a4a76294778c99600228091e797507026d8ba4c3"},"cell_type":"markdown"},{"source":"num_classes = 5270\nnum_train_images = len(train_images_df)\nnum_val_images = len(val_images_df)\nbatch_size = 128\n\n# Tip: use ImageDataGenerator for data augmentation and preprocessing.\ntrain_datagen = ImageDataGenerator()\ntrain_gen = BSONIterator(train_bson_file, train_images_df, train_offsets_df, \n                         num_classes, train_datagen, lock,\n                         batch_size=batch_size, shuffle=True)\n\nval_datagen = ImageDataGenerator()\nval_gen = BSONIterator(train_bson_file, val_images_df, train_offsets_df,\n                       num_classes, val_datagen, lock,\n                       batch_size=batch_size, shuffle=True)","outputs":[],"metadata":{"_cell_guid":"d9381773-ad58-4656-b225-306e68a603f7","_uuid":"ffc317a85c6849dca92c8e5322eecff66269cca4"},"execution_count":null,"cell_type":"code"},{"source":"How fast is the generator? Create a single batch:","metadata":{"_cell_guid":"5c5871a3-6d4a-4260-b8d4-462454ad8983","_uuid":"c3cb7cadf70e26f22f9cd35f8f375b4e9483325a"},"cell_type":"markdown"},{"source":"next(train_gen)  # warm-up\n\n%time bx, by = next(train_gen)","outputs":[],"metadata":{"_cell_guid":"1b3227e9-902a-41ff-a16a-460c3eb5c7e3","_uuid":"e5092cb6348a56b0f67d5a0b08115125eab2774a"},"execution_count":null,"cell_type":"code"},{"source":"Does it really output images and one-hot encoded class labels? Note that the images are pre-processed (and augmented) and therefore may look weird.","metadata":{"_cell_guid":"b393341b-1ab5-4f1f-9cfc-06cd2537d993","_uuid":"98f84fd5e6a320f891b9ac21b345e2a590d37c82"},"cell_type":"markdown"},{"source":"plt.imshow(bx[-1].astype(np.uint8))","outputs":[],"metadata":{"_cell_guid":"f2c3d3d3-f030-4b7c-b1d5-bf918d17f32f","_uuid":"4dbc717fd62d116085f7b17092fb0f785e38b03e"},"execution_count":null,"cell_type":"code"},{"source":"cat_idx = np.argmax(by[-1])\ncat_id = idx2cat[cat_idx]\ncategories_df.loc[cat_id]","outputs":[],"metadata":{"_cell_guid":"9bafa70a-6f48-4c08-a291-213e276c96e4","_uuid":"a1567274363fc542693d44e68385c77414c5528a"},"execution_count":null,"cell_type":"code"},{"source":"%time bx, by = next(val_gen)","outputs":[],"metadata":{"_cell_guid":"52a5f50b-c334-47b7-9696-04930acacc70","_uuid":"dc592b117db8c985f51e5846510775f68f9e597f"},"execution_count":null,"cell_type":"code"},{"source":"plt.imshow(bx[-1].astype(np.uint8))","outputs":[],"metadata":{"_cell_guid":"010217b8-7ebb-4396-bbdc-bf5d41913bbf","_uuid":"a5d562f80ab3d820d1585888f775250b95dc9a59"},"execution_count":null,"cell_type":"code"},{"source":"cat_idx = np.argmax(by[-1])\ncat_id = idx2cat[cat_idx]\ncategories_df.loc[cat_id]","outputs":[],"metadata":{"_cell_guid":"95881156-e36f-4209-ba52-ba871dd267e1","_uuid":"4be78cd87a23367b4ed07007ca15b5fda6c5b877"},"execution_count":null,"cell_type":"code"},{"source":"# Part 3: Training\n\nCreate a very simple Keras model and train it, to test that the generators work.","metadata":{"_cell_guid":"2250dba6-6ab0-4417-854e-6a3eb924fc33","_uuid":"3daa4847924e17f4bab9a58470460667dc517075"},"cell_type":"markdown"},{"source":"from keras.models import Sequential\nfrom keras.layers import Dropout, Flatten, Dense\nfrom keras.layers.convolutional import Conv2D\nfrom keras.layers.pooling import MaxPooling2D, GlobalAveragePooling2D\n\nmodel = Sequential()\nmodel.add(Conv2D(32, 3, padding=\"same\", activation=\"relu\", input_shape=(180, 180, 3)))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(64, 3, padding=\"same\", activation=\"relu\"))\nmodel.add(MaxPooling2D())\nmodel.add(Conv2D(128, 3, padding=\"same\", activation=\"relu\"))\nmodel.add(MaxPooling2D())\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(num_classes, activation=\"softmax\"))\n\nmodel.compile(optimizer=\"adam\",\n              loss=\"categorical_crossentropy\",\n              metrics=[\"accuracy\"])\n\nmodel.summary()","outputs":[],"metadata":{"_cell_guid":"eaa3b3ba-7502-445f-91da-26d29af2bee8","_uuid":"126271374bf7cfcf0871b8b78441e1f1dfa05cbb"},"execution_count":null,"cell_type":"code"},{"source":"# To train the model:\nmodel.fit_generator(train_gen,\n                    steps_per_epoch = 10,   #num_train_images // batch_size,\n                    epochs = 3,\n                    validation_data = val_gen,\n                    validation_steps = 10,  #num_val_images // batch_size,\n                    workers = 8)","outputs":[],"metadata":{"_kg_hide-output":false,"_cell_guid":"0b8d0061-a07c-4a78-b53c-6ca1a2cf9799","_uuid":"ce48a33358e9824f2c19e2c69878be9762915b1c"},"execution_count":null,"cell_type":"code"},{"source":"# To evaluate on the validation set:\n#model.evaluate_generator(val_gen, steps=num_val_images // batch_size, workers=8)","outputs":[],"metadata":{"_cell_guid":"d2822e2c-6b4e-4b8b-b346-86e4b5feeb42","collapsed":true,"_uuid":"3303ec963fa2aeef322fd9d864b296ba1bbb04b1"},"execution_count":null,"cell_type":"code"},{"source":"# Part 4: Test set predictions\n\nNote: The previous version of this kernel used `BSONIterator` to load the test set images in batches. However, storing the prediction results takes up a huge amount of memory. \n\nI suggest using a different kind of generator instead, something like the following:\n\n```\nfrom keras import backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\n\nsubmission_df = pd.read_csv(data_dir + \"sample_submission.csv\")\nsubmission_df.head()\n\ntest_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)\ndata = bson.decode_file_iter(open(test_bson_path, \"rb\"))\n\nwith tqdm(total=num_test_products) as pbar:\n    for c, d in enumerate(data):\n        product_id = d[\"_id\"]\n        num_imgs = len(d[\"imgs\"])\n\n        batch_x = np.zeros((num_imgs, 180, 180, 3), dtype=K.floatx())\n\n        for i in range(num_imgs):\n            bson_img = d[\"imgs\"][i][\"picture\"]\n\n            # Load and preprocess the image.\n            img = load_img(io.BytesIO(bson_img), target_size=(180, 180))\n            x = img_to_array(img)\n            x = test_datagen.random_transform(x)\n            x = test_datagen.standardize(x)\n\n            # Add the image to the batch.\n            batch_x[i] = x\n\n        prediction = model.predict(batch_x, batch_size=num_imgs)\n        avg_pred = prediction.mean(axis=0)\n        cat_idx = np.argmax(avg_pred)\n        \n        submission_df.iloc[c][\"category_id\"] = idx2cat[cat_idx]        \n        pbar.update()\n        \nsubmission_df.to_csv(\"my_submission.csv.gz\", compression=\"gzip\", index=False)        \n```","metadata":{"_cell_guid":"2116a8c3-ec60-4878-93af-94c185c08cb3","_uuid":"ee05e14fd2e9354008f23941f97ee3ff7ff73aaf"},"cell_type":"markdown"},{"source":"","outputs":[],"metadata":{"_cell_guid":"cbfc7694-29b3-4197-b2dd-daca30ed7468","collapsed":true,"_uuid":"63f279cfa4548b449061ab932e15005d1ee59e26"},"execution_count":null,"cell_type":"code"}],"nbformat_minor":1,"metadata":{"language_info":{"file_extension":".py","mimetype":"text/x-python","name":"python","codemirror_mode":{"name":"ipython","version":3},"version":"3.6.1","pygments_lexer":"ipython3","nbconvert_exporter":"python"},"kernelspec":{"display_name":"Python 3","name":"python3","language":"python"}}}