{"metadata": {"language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "version": "3.6.3", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "mimetype": "text/x-python"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "cells": [{"metadata": {"_cell_guid": "75d61c38-954c-4bd1-a127-5d17e1812402", "_uuid": "2ae1f395d8768512da0739dded661e329dbfa14e"}, "source": ["This notebook uses the first part of[ Human Analog Kernel](https://www.kaggle.com/humananalog/keras-generator-for-reading-directly-from-bson) to save train imagens as two independent binary files for train and vallidation. Then define a generator class for Keras that iterate over the bin files thread safely. This approach tends to be faster than reading from BSON file, as we no more need to do random access. On the other hand, has the draw back to momentanealy double the space used to store images."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "4036f11e-5223-46e5-8948-5cbf70a9bf73", "_uuid": "3767d2738682e30c292a466f66bc75fcc80a5076", "scrolled": true}, "outputs": [], "cell_type": "code", "source": ["import os, sys, math, io\n", "import numpy as np\n", "import pandas as pd\n", "import multiprocessing as mp\n", "import bson\n", "import struct\n", "\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "\n", "import keras\n", "from keras.preprocessing.image import load_img, img_to_array\n", "import tensorflow as tf\n", "\n", "from collections import defaultdict\n", "from tqdm import *\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "3601ddc2-5dca-463c-be54-a297e019469a", "_uuid": "41fb122f53b29ff6c7cdba4f08001fa8204fab34"}, "outputs": [], "cell_type": "code", "source": ["keras.__version__, tf.__version__"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "dfec6cb3-3b14-44cc-920c-c4bba4f79589", "_uuid": "1d92f3f86af7906e204eec1d536bd29878fed02d"}, "outputs": [], "cell_type": "code", "source": ["data_dir = \"../input/\"\n", "\n", "train_bson_path = os.path.join(data_dir, \"train.bson\")\n", "num_train_products = 7069896\n", "\n", "# train_bson_path = os.path.join(data_dir, \"train_example.bson\")\n", "# num_train_products = 82\n", "\n", "test_bson_path = os.path.join(data_dir, \"test.bson\")\n", "num_test_products = 1768182"]}, {"metadata": {"_cell_guid": "3c24b66f-0f55-4007-86d9-aae6ec1c16bc", "_uuid": "0c1457925b72937cc9b8d431142f06098cbfe06c"}, "source": ["# Part 1: Create lookup tables (Credits: Human Analog)\n", "\n", "The generator uses several lookup tables that describe the layout of the BSON file, which products and images are part of the training/validation sets, and so on.\n", "\n", "You only need to generate these tables once, as they get saved to CSV files. If you already have these CSV files, skip to part 2."], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "593c6f49-83eb-4491-bcd1-0131845e395c", "_uuid": "7ea59756f0ab3eb271e2b5d6495e6a158311de35"}, "source": ["## Lookup table for categories"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "c049b397-8b31-4389-9657-f0281c5f7b4f", "_uuid": "c2a19dc1ea89274a1ab3332fa635e6dfdc385ce6"}, "outputs": [], "cell_type": "code", "source": ["categories_path = os.path.join(data_dir, \"category_names.csv\")\n", "categories_df = pd.read_csv(categories_path, index_col=\"category_id\")\n", "\n", "# Maps the category_id to an integer index. This is what we'll use to\n", "# one-hot encode the labels.\n", "categories_df[\"category_idx\"] = pd.Series(range(len(categories_df)), index=categories_df.index)\n", "\n", "categories_df.to_csv(\"categories.csv\")\n", "categories_df.head()"]}, {"metadata": {"_cell_guid": "92491e45-56da-4d64-a887-6cf7a8c8cd30", "_uuid": "05ccbed26b3518d90ddcc9c26668f6008b89cb0f"}, "source": ["Create dictionaries for quick lookup of `category_id` to `category_idx` mapping."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7aad702e-68b9-4c94-a6e7-910283629011", "_uuid": "1c40f20530ad22246d6941845c89b210b4a85368"}, "outputs": [], "cell_type": "code", "source": ["def make_category_tables():\n", "    cat2idx = {}\n", "    idx2cat = {}\n", "    for ir in categories_df.itertuples():\n", "        category_id = ir[0]\n", "        category_idx = ir[4]\n", "        cat2idx[category_id] = category_idx\n", "        idx2cat[category_idx] = category_id\n", "    return cat2idx, idx2cat"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "99c21338-72c9-406a-bc3a-ba33d7310086", "_uuid": "b68af0dda86d6c6af127a5bf634778fb63bda958"}, "outputs": [], "cell_type": "code", "source": ["cat2idx, idx2cat = make_category_tables()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "530dac1b-2891-42ff-ab7f-ea1fe8ef01e6", "_uuid": "3fff32a945788e89e676c68fc20ff138ac72f754"}, "outputs": [], "cell_type": "code", "source": ["# Test if it works:\n", "cat2idx[1000012755], idx2cat[4]"]}, {"metadata": {"_cell_guid": "d4518fba-7f26-45c1-ae3f-daf72ffbbee9", "_uuid": "51f82526d1ce6b7ab0b4d5b370f0f2f9601dbc66"}, "source": ["## Read the BSON files\n", "\n", "We store the offsets and lengths of all items, allowing us random access to the items later.\n", "\n", "Inspired by code from: https://www.kaggle.com/vfdev5/random-item-access\n", "\n", "Note: this takes a few minutes to execute, but we only have to do it once (we'll save the table to a CSV file afterwards)."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "2419b6a8-d141-4f5d-a868-778e0483fc40", "_uuid": "50f1fa1bc0a9c0597ce90d7c7daf2d0b7a788467"}, "outputs": [], "cell_type": "code", "source": ["def read_bson(bson_path, num_records, with_categories):\n", "    rows = {}\n", "    with open(bson_path, \"rb\") as f, tqdm(total=num_records) as pbar:\n", "        offset = 0\n", "        while True:\n", "            item_length_bytes = f.read(4)\n", "            if len(item_length_bytes) == 0:\n", "                break\n", "\n", "            length = struct.unpack(\"<i\", item_length_bytes)[0]\n", "\n", "            f.seek(offset)\n", "            item_data = f.read(length)\n", "            assert len(item_data) == length\n", "\n", "            item = bson.BSON.decode(item_data)\n", "            product_id = item[\"_id\"]\n", "            num_imgs = len(item[\"imgs\"])\n", "\n", "            row = [num_imgs, offset, length]\n", "            if with_categories:\n", "                row += [item[\"category_id\"]]\n", "            rows[product_id] = row\n", "\n", "            offset += length\n", "            f.seek(offset)\n", "            pbar.update()\n", "\n", "    columns = [\"num_imgs\", \"offset\", \"length\"]\n", "    if with_categories:\n", "        columns += [\"category_id\"]\n", "\n", "    df = pd.DataFrame.from_dict(rows, orient=\"index\")\n", "    df.index.name = \"product_id\"\n", "    df.columns = columns\n", "    df.sort_index(inplace=True)\n", "    return df"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "54c03644-11a0-425b-bf2e-b412f087a390", "_uuid": "3607b538f418145677d61bc80f6a0b6e97bd1c95"}, "outputs": [], "cell_type": "code", "source": ["%time train_offsets_df = read_bson(train_bson_path, num_records=num_train_products, with_categories=True)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "4e6d4a2e-2014-4610-b624-0a5de1ae803e", "_uuid": "507ae4e5fceec73fdfb08494e0dcde86b0d62a29"}, "outputs": [], "cell_type": "code", "source": ["train_offsets_df.head()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "8d24b1ed-8f75-4e29-a70d-627ae11deb45", "_uuid": "3e5daf1253397d05065568ad3ce69be73042cc59"}, "outputs": [], "cell_type": "code", "source": ["train_offsets_df.to_csv(\"train_offsets.csv\")"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "0c331f10-e2af-43ee-8bcd-1b4058430cbd", "_uuid": "254a22b36211385bc943f1c3d10914f75548b5e0"}, "outputs": [], "cell_type": "code", "source": ["# How many products?\n", "len(train_offsets_df)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "c357322f-5026-4454-9c0f-a2d40b72c99a", "_uuid": "9aa11d00937321bcfe68f80be0cc17a2cab39e65"}, "outputs": [], "cell_type": "code", "source": ["# How many categories?\n", "len(train_offsets_df[\"category_id\"].unique())"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7eac8e95-7930-4e07-a869-5fcd0eb8e1a1", "_uuid": "f1ed766c559b2b7848e452b0b94421c3d4767c81"}, "outputs": [], "cell_type": "code", "source": ["# How many images in total?\n", "train_offsets_df[\"num_imgs\"].sum()"]}, {"metadata": {"_cell_guid": "663b73b1-4de8-4adc-98b8-475403bd752d", "_uuid": "39e87235e9b8416ccbf6ea0909be65cfe6d00b65"}, "source": ["## Create a random train/validation split\n", "\n", "We split on products, not on individual images. Since some of the categories only have a few products, we do the split separately for each category.\n", "\n", "This creates two new tables, one for the training images and one for the validation images. There is a row for every single image, so if a product has more than one image it occurs more than once in the table."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "24e8db94-3a2b-453d-a8ac-1de2460ed129", "_uuid": "4d702f77af77901c7ed547eebb1da9146afea667"}, "outputs": [], "cell_type": "code", "source": ["def make_val_set(df, split_percentage=0.2, drop_percentage=0.):\n", "    # Find the product_ids for each category.\n", "    category_dict = defaultdict(list)\n", "    for ir in tqdm(df.itertuples()):\n", "        category_dict[ir[4]].append(ir[0])\n", "\n", "    train_list = []\n", "    val_list = []\n", "    with tqdm(total=len(df)) as pbar:\n", "        for category_id, product_ids in category_dict.items():\n", "            category_idx = cat2idx[category_id]\n", "\n", "            # Randomly remove products to make the dataset smaller.\n", "            keep_size = int(len(product_ids) * (1. - drop_percentage))\n", "            if keep_size < len(product_ids):\n", "                product_ids = np.random.choice(product_ids, keep_size, replace=False)\n", "\n", "            # Randomly choose the products that become part of the validation set.\n", "            val_size = int(len(product_ids) * split_percentage)\n", "            if val_size > 0:\n", "                val_ids = np.random.choice(product_ids, val_size, replace=False)\n", "            else:\n", "                val_ids = []\n", "\n", "            # Create a new row for each image.\n", "            for product_id in product_ids:\n", "                row = [product_id, category_idx]\n", "                for img_idx in range(df.loc[product_id, \"num_imgs\"]):\n", "                    if product_id in val_ids:\n", "                        val_list.append(row + [img_idx])\n", "                    else:\n", "                        train_list.append(row + [img_idx])\n", "                pbar.update()\n", "                \n", "    columns = [\"product_id\", \"category_idx\", \"img_idx\"]\n", "    train_df = pd.DataFrame(train_list, columns=columns)\n", "    val_df = pd.DataFrame(val_list, columns=columns)   \n", "    return train_df, val_df"]}, {"metadata": {"_cell_guid": "8df853cd-b2e5-4c05-8162-e332c5995bdc", "_uuid": "6e3c437a8169a31e9e4f4170f70d48668f8f0934"}, "source": ["Create a 80/20 split. Also drop 90% of all products to make the dataset more manageable. (Note: if `drop_percentage` > 0, the progress bar doesn't go all the way.)"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "a27d017f-8ac2-4480-8b48-7d092be8de82", "_uuid": "586a0ad09c6ce8af090277d0ac930ba53eb3a8d3"}, "outputs": [], "cell_type": "code", "source": ["train_images_df, val_images_df = make_val_set(train_offsets_df, split_percentage=0.2, \n", "                                              drop_percentage=0.9)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "23272dc8-21b2-4880-b7b7-fafb4ce523d0", "_uuid": "c45a37b5a02e07a3c095feea9012fc501fdf820b"}, "outputs": [], "cell_type": "code", "source": ["train_images_df.head()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "c3e170c3-4f7a-499a-8bca-22f5e6a20693", "_uuid": "8b7255493416498f9151645648c560744187da03"}, "outputs": [], "cell_type": "code", "source": ["val_images_df.head()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "0cb17194-01c4-4b38-a140-ec8330e7aaa8", "_uuid": "5b9ee034409bed1eeafe54bceb3c008c4d4c9334"}, "outputs": [], "cell_type": "code", "source": ["print(\"Number of training images:\", len(train_images_df))\n", "print(\"Number of validation images:\", len(val_images_df))\n", "print(\"Total images:\", len(train_images_df) + len(val_images_df))"]}, {"metadata": {"_cell_guid": "f0b89d28-b068-44d2-909d-d43e0dfe1ab2", "_uuid": "7aaae46c2b9b773cfee6430e4049c7abe0b69af9"}, "source": ["Are all categories represented in the train/val split? (Note: if the drop percentage is high, then very small categories won't have enough products left to make it into the validation set.)"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "9685e057-5606-4e73-9ab1-33e2d638faa1", "_uuid": "5abbe4cf931245641be378d677963dde40800cb7"}, "outputs": [], "cell_type": "code", "source": ["len(train_images_df[\"category_idx\"].unique()), len(val_images_df[\"category_idx\"].unique())"]}, {"metadata": {"_cell_guid": "59abf374-4259-4e2b-8bcb-f75947f9d4dc", "_uuid": "8090c3dfdf328ea0e9f89b2280126a5a58840c09"}, "source": ["Quickly verify that the split really is approximately 80-20:"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "5666aad4-577d-431e-a26c-39b289abc3da", "_uuid": "1bdbdfe266a8c4510a51f89896aa3c9ad17d9f72"}, "outputs": [], "cell_type": "code", "source": ["category_idx = 619\n", "num_train = np.sum(train_images_df[\"category_idx\"] == category_idx)\n", "num_val = np.sum(val_images_df[\"category_idx\"] == category_idx)\n", "num_val / num_train"]}, {"metadata": {"_cell_guid": "624c622f-059a-4ce0-b68e-f373dba30b03", "_uuid": "1bb825a934deaa8a287a0243927193377e6fa433"}, "source": ["Close enough. ;-) Remember that we split on products but not all products have the same number of images, which is where the slightly discrepancy comes from. (Also, there tend to be fewer validation images if `drop_percentage` > 0.)"], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "f84f672e-c16d-47a7-9d1c-84f9a0fb3e22", "_uuid": "9450109fb5c910c9e7634855b7445fc4260097a7"}, "source": ["Save the lookup tables as CSV so that we don't need to repeat the above procedure again."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "c880c70f-dd98-4d61-b65f-f230abffdefd", "_uuid": "747743164ea6dae14a5de4eb624465cb806089ab"}, "outputs": [], "cell_type": "code", "source": ["train_images_df.to_csv(\"train_images.csv\")\n", "val_images_df.to_csv(\"val_images.csv\")"]}, {"metadata": {"_cell_guid": "4212f62a-86c3-4a13-8882-b4c90f64dac5", "_uuid": "8a4bd65c90576d611340e36bce5f49896d942ed1"}, "source": ["# Part 2: Creating the new bin files"], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "1da50467-d11d-417e-9367-bd8574dfe61a", "_uuid": "55331a57195739250c5b530160cf32a57b9a2464"}, "source": ["First load the lookup tables from the CSV files (you don't need to do this if you just did all the steps from part 1)."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "19c24f98-551d-4b95-b8f7-8497ab09eb56", "_uuid": "0c0dadb16f6b094748e967e9c682bb18b3b5c336"}, "outputs": [], "cell_type": "code", "source": ["categories_df = pd.read_csv(\"categories.csv\", index_col=0)\n", "cat2idx, idx2cat = make_category_tables()\n", "\n", "train_offsets_df = pd.read_csv(\"train_offsets.csv\", index_col=0)\n", "train_images_df = pd.read_csv(\"train_images.csv\", index_col=0)\n", "val_images_df = pd.read_csv(\"val_images.csv\", index_col=0)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "11cde6b7-7c05-4f7c-9c91-2973b6482195", "_uuid": "ff20d022a8062d03cd712601cb1b1ee961cbb17d"}, "outputs": [], "cell_type": "code", "source": ["train_offsets_df.head()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7a09d155-bc21-4dda-bad0-06762c3589cd", "_uuid": "e2e8b11300d8d3f7dccfc52de0e7cb3c045521d0"}, "outputs": [], "cell_type": "code", "source": ["train_images_df.head()"]}, {"metadata": {"_cell_guid": "50561348-9ad6-4a80-b92f-6746897da4c1", "_uuid": "776c5a1bce9de0e79b773959ff78de2e8e243a45"}, "source": ["**First change**: lets shuffle the train images before creating of bin file..."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "f86277ad-48a7-463a-8614-6b39bd6f7995", "_uuid": "98fe9a58423b7153dbead69986975081fe8fa9ef"}, "outputs": [], "cell_type": "code", "source": ["train_images_df = train_images_df.sample(frac=1).reset_index(drop=True)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "945de056-a56a-4380-9906-ab0520aeff6f", "_uuid": "0919782cbc3981abbc9a389127f5c8aef264e76e"}, "outputs": [], "cell_type": "code", "source": ["train_images_df.head()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "3bcd0bf2-e268-40b7-96f5-35229ae59be0", "_uuid": "91f17fcffd0f92e21b09c23386146a83ac3872c9"}, "outputs": [], "cell_type": "code", "source": ["val_images_df.head()"]}, {"metadata": {"_cell_guid": "17cb7bb4-4941-4d0d-8fa5-2dd60b7014ce", "_uuid": "edbb272356d65e3ec9b540bd4597464ae4afa3c5"}, "source": ["We will use Human Analogs's index dataframes  as a source for our 2 new binary files. One for train and the other for validation. These bin files will be used by the new data generator \n", "\n", "**Note:** For fastest results, put the train.bson and test.bson files on a fast drive (SSD).\n"], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "7d027334-46d0-4486-90a4-7350edac316e", "_uuid": "b1c30a6463448aa30c9bb9768f5c965b9e8b5cc6"}, "source": ["Lets use Label Encoder for the categories"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "62ffb40a-b78d-4404-9731-64bcaefaea9b", "_uuid": "8adb4c654c6301497e2e9f56a6c1386750d21469"}, "outputs": [], "cell_type": "code", "source": ["#Uses LabelEncoder for class_id encoding\n", "from sklearn import preprocessing\n", "le = preprocessing.LabelEncoder()\n", "le.fit(pd.read_csv(categories_path).category_id)"]}, {"metadata": {"_cell_guid": "68e3b62f-265a-4ebd-8c99-63a962c1cf02", "_uuid": "37dbd82ca34760b8db4c47ecd664d1f60165b6df"}, "source": ["Lets test it..."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7efc8f60-0d91-4daa-a000-d757a21b85cc", "_uuid": "272481a6706b516590059d3ee6ca8d6b0818aaab"}, "outputs": [], "cell_type": "code", "source": ["#Testing the encoder\n", "original=le.classes_[:5]\n", "print(\"5 original classes:\", original)\n", "encoded=le.transform(original)\n", "print(\"5 encoded classes:\",encoded)\n", "print(\"getting back the original classes:\", le.inverse_transform(encoded))\n"]}, {"metadata": {"_cell_guid": "c41028f4-7679-48ad-bb25-9ca3dd3bf5c2", "_uuid": "2313e24b8fede9d7d3c3e296e614f8e75d04c56d"}, "source": ["Seems fine!\n", "Now let's define a function for creating the new binaries files and another function to test it."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "dcd6b7bc-81f2-463e-9a0a-36af9da05795", "_uuid": "9eb21c2cfeb2164c48fdcc510d3aeff16b70434d"}, "outputs": [], "cell_type": "code", "source": ["def create_bin_file(images_df, offsets_df, bson_file_name, bin_file_name, encoder):\n", "    with open(bson_file_name, 'rb') as bson_file, open(bin_file_name, 'wb') as bin_file:    \n", "        #uses Human Analog previously created dataframes\n", "        for index, row in images_df.iterrows():\n", "            offset_row = offsets_df.loc[row.product_id]\n", "            bson_file.seek(offset_row[\"offset\"])\n", "            item_data = bson_file.read(offset_row[\"length\"])\n", "\n", "            # Grab the image from the product.\n", "            item = bson.BSON.decode(item_data)\n", "            img_idx = row[\"img_idx\"]\n", "            bson_img = item[\"imgs\"][img_idx][\"picture\"]\n", "\n", "            #write down the encoded class, the size of the img and the img it self \n", "            encoded_class = encoder.transform([offset_row.category_id])[0]\n", "            img_size = len(bson_img)\n", "            bin_file.write(struct.pack('<ii', encoded_class, img_size))   \n", "            bin_file.write(bytes(bson_img))   \n", "        bin_file.close()\n", "        bson_file.close()"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "768dcecd-73e8-4e34-9521-53a2cd66250f", "_uuid": "d0255a886815d0d203f201a450503a164d32f5e3"}, "outputs": [], "cell_type": "code", "source": ["#test function\n", "def bin_file_test(file_name, encoder, n=3):\n", "    with open(file_name, 'rb') as bin_file:    \n", "        count = 0\n", "        while count<n:\n", "            count += 1 \n", "            buffer=bin_file.read(8)\n", "            encoded_class, length = struct.unpack(\"<ii\", buffer)\n", "            bson_img = bin_file.read(length)\n", "            img = load_img(io.BytesIO(bson_img), target_size=(180,180))\n", "            plt.figure()\n", "            plt.imshow(img)\n", "            plt.text(5,20,\n", "                     \"%d Class: %s (size: %d)\" %(count, encoder.inverse_transform(encoded_class), length),\n", "                    backgroundcolor='0.75',alpha=.5)\n"]}, {"metadata": {"_cell_guid": "398da859-f37a-4e55-9d78-14cc48d7fef8", "_uuid": "439606cd16fe163d0d0aaaddb44317ab8ebb156e"}, "source": ["Lets create the train bin file and test it. Because of kernel limits we will only write down 1000 images. In production enviroment you should remove this limit. "], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "1ba0bbcb-2dda-47d1-a0ae-613e597ba5cc", "_uuid": "c4dfb595e52c2dd3137da3d29fe5a2c8f9518687"}, "outputs": [], "cell_type": "code", "source": ["#create train bin file and test it!!!\n", "img_df = train_images_df[:1000] #remove this in production environment\n", "create_bin_file(img_df, train_offsets_df, train_bson_path, 'train.bin', le)\n", "bin_file_test('train.bin', le, n=9)"]}, {"metadata": {"_cell_guid": "842b94dc-4fff-4733-895b-59d42121cae8", "_uuid": "15bbe11de5a3f3ebf11f505d0eafd6b35983f27d"}, "source": ["So sweet!!! \n", "Now lets create the validation bin file and test it too. Because of kernel limits we will only write down 1000 images. In production enviroment you should remove this limit. "], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "de679c1d-ad7d-4db5-8ff9-e5f8d9bcc83a", "_uuid": "684f7de542c98ad705c8103aabcb09a5eac9078a"}, "outputs": [], "cell_type": "code", "source": ["#create val bin file and test it\n", "img_df = val_images_df[:1000] #remove this in production environment\n", "create_bin_file(img_df, train_offsets_df, train_bson_path, 'val.bin', le)\n", "bin_file_test('val.bin', le)"]}, {"metadata": {"_cell_guid": "88d78467-a398-4264-9161-2c847d09fb2d", "_uuid": "2ba6e9102081b25aebcce0da3b853279a4d6657f"}, "source": ["Beautifull!!!\n", "The last step is to define the binary file data generator."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "cd468f60-f39e-40e5-9b91-374b85368de6", "_uuid": "9ea3b00811fa7c118861390b3125235ae35960c7"}, "outputs": [], "cell_type": "code", "source": ["from keras.preprocessing import image\n", "from keras.preprocessing.image import Iterator\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras import backend as K\n", "import threading\n", "\n", "#The generator. The flow method does the generator job!\n", "class BinFileIterator(Iterator):\n", "    def __init__(self, bin_file_name, img_generator, samples, \n", "                 target_size=(180,180), \n", "                 batch_size=32, num_class=5270):\n", "        self.file = open(bin_file_name,'rb')\n", "        self.img_gen=img_generator\n", "        self.target_size = tuple(target_size)\n", "        self.image_shape = self.target_size + (3,)\n", "        self.num_class = num_class\n", "        self.lock = threading.Lock() #Since we have 2 files, each generator has its own lock\n", "        super(BinFileIterator, self).__init__(samples, batch_size, shuffle=False, seed=None)\n", "\n", "    def flow(self, index_array):\n", "        X = np.zeros((len(index_array),) + self.image_shape, dtype=K.floatx())\n", "        Y = np.zeros((len(index_array), self.num_class), dtype=K.floatx())\n", "\n", "        for i, j in enumerate(index_array):\n", "            with self.lock:\n", "                buffer=self.file.read(8)\n", "                if len(buffer) < 8:\n", "                    self.file.seek(0)\n", "                    buffer=self.file.read(8)\n", "                encoded_class, length = struct.unpack(\"<ii\", buffer)\n", "                bson_img = self.file.read(length)\n", "            img = load_img(io.BytesIO(bson_img), target_size=self.target_size)\n", "            x = image.img_to_array(img)\n", "            x = self.img_gen.random_transform(x)\n", "            x = self.img_gen.standardize(x)\n", "            X[i] = x\n", "            Y[i, encoded_class] = 1\n", "        return X, Y\n", "\n", "    def next(self):\n", "        with self.lock: \n", "            index_array = next(self.index_generator)\n", "        return self.flow(index_array[0])        \n"]}, {"metadata": {"_cell_guid": "a729a24f-f926-464e-8790-e549f327fb4d", "_uuid": "ed7c7179491b6d4218dd21f1fc21f7f21a753984"}, "source": ["## Testing\n", "Lets see if the iterator is working. Lets get 3 batches of 3 images, totaling 9 images."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "70a61d68-51be-46ca-bec3-1eec81882b61", "_uuid": "cd3a6285c4d514a28fbac5282cbe6cd58faf2e0a"}, "outputs": [], "cell_type": "code", "source": ["train_img_gen = ImageDataGenerator()\n", "data_gen = BinFileIterator('train.bin', img_generator=train_img_gen,  samples=1000,\n", "                 target_size=(180,180), \n", "                 batch_size=3)"]}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "8f93a132-25e0-439c-b9a3-640f3ec28762", "_uuid": "976981f8a725183547fb176c1f3727cf65b562e2"}, "outputs": [], "cell_type": "code", "source": ["for b in range(3):\n", "  imgs, categories = data_gen.next()\n", "  for img, category in zip(imgs, categories): \n", "      plt.figure()\n", "      plt.imshow(img)\n", "      plt.text(5,20,\n", "               \"Class: %d %s\" % (np.argmax(category), le.inverse_transform(np.argmax(category))),\n", "               backgroundcolor='0.75',alpha=.5)"]}, {"metadata": {"_cell_guid": "d290b8c2-011c-4a00-98b8-7fd12ee0f365", "_uuid": "9ab4b83652f1c18fb3604b903a875b55e1c61d4a"}, "source": ["We see that the first 9 imagens are being correctly retrieved. Note that they are standardized.\n", "Now lets do a performance test. lets retrieve 10 batches of 128 images and mesure the execution time. Note that we only have 1000 images, so the iterator must start over."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "38309fd4-4897-44dc-929d-74e8154d67ae", "_uuid": "6276b88687be35c437acf308a13aad3b85c3e2a3"}, "outputs": [], "cell_type": "code", "source": ["import time\n", "data_gen = BinFileIterator('train.bin', img_generator=train_img_gen,  samples=1000,\n", "                 target_size=(180,180), \n", "                 batch_size=128) #We changed the batch size here \n", "for b in range(3):\n", "  %time imgs, categories = data_gen.next()\n", "  print(\"Retrieved: %d\" %(len(imgs))  ) \n"]}, {"metadata": {"_cell_guid": "2ba82a36-62a3-4b35-8a60-f415f32ad2ba", "_uuid": "5074444f359c10ecde01ef083a7257da1d0b116c"}, "source": ["Lets take a look at the last image:"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "62b0cd70-04a2-4dc8-a390-161e15b29a3c", "_uuid": "691cc51c974b558ca946c9d1ddfd6959c46ee6de"}, "outputs": [], "cell_type": "code", "source": ["plt.figure()\n", "plt.imshow(imgs[-1])\n", "plt.text(5,20,\n", "        \"Class: %d %s\" % (np.argmax(categories[-1]), le.inverse_transform(np.argmax(categories[-1]))),\n", "        backgroundcolor='0.75',alpha=.5)"]}, {"metadata": {"_cell_guid": "02486182-3258-4d88-95f1-b341d8b9a0b2", "_uuid": "3d68edc9a291316b5fee60f925bacd29091eb173"}, "source": ["We have got an reduction in execution time from 315ms to about 130. It means an improvement of 50% \n", "Now lets see if this generator is thread safe."], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7513dae4-3f22-42b2-a8af-ce582e8e81dd", "_uuid": "54de25ebda8f5583939f3857a3b2aca0430936c6"}, "outputs": [], "cell_type": "code", "source": ["import _thread\n", "\n", "\n", "#Lets use a large batch size\n", "data_gen = BinFileIterator('train.bin', img_generator=train_img_gen,  samples=1000,\n", "                 target_size=(180,180), \n", "                 batch_size=450) #We changed the batch size here \n", "\n", "# Define a function for the thread\n", "def execute_batch(t_name):\n", "   imgs, categories = data_gen.next()\n", "   print(t_name, \"retrieved: %d\" %len(imgs), \n", "                 \"last category:\" , le.inverse_transform(np.argmax(categories[-1])))\n", "\n", "# Create two threads as follows\n", "try:\n", "   _thread.start_new_thread( execute_batch, (\"Thread-1\", ) )\n", "   time.sleep(0.001)   \n", "   _thread.start_new_thread( execute_batch, (\"Thread-2\", ) )\n", "   time.sleep(0.001)   \n", "   _thread.start_new_thread( execute_batch, (\"Thread-3\", ) )\n", "except:\n", "   print (\"Error: unable to start thread\")\n", "\n", "time.sleep(5)"]}, {"metadata": {"_cell_guid": "61175bf7-6de2-4811-9057-40e4ffe4deeb", "_uuid": "169b6c9d0d26ba3204e5df6e4dc146e6fc88db81"}, "source": ["It seems that every thing is working fine. Last thread finished first, as expect, as it retrieves less images."], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "b9b075ea-73b1-42e7-9c79-b03a1ae796b3", "_uuid": "46974382c74fb17e98ace18119f40a595dd5544f"}, "source": ["# Training procedure\n", "The training procedure is pretty much the same, except for the iterator creation:"], "cell_type": "markdown"}, {"execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "b10cc1fd-5625-4899-8d49-9db75f611a29", "_uuid": "c1e6ed653175870214bac7670ba9d3222734b908"}, "outputs": [], "cell_type": "code", "source": ["from keras.models import Sequential\n", "from keras.layers import Dropout, Flatten, Dense\n", "from keras.layers.convolutional import Conv2D\n", "from keras.layers.pooling import MaxPooling2D, GlobalAveragePooling2D\n", "\n", "model = Sequential()\n", "model.add(Conv2D(32, 3, padding=\"same\", activation=\"relu\", input_shape=(180, 180, 3)))\n", "model.add(MaxPooling2D())\n", "model.add(Conv2D(64, 3, padding=\"same\", activation=\"relu\"))\n", "model.add(MaxPooling2D())\n", "model.add(Conv2D(128, 3, padding=\"same\", activation=\"relu\"))\n", "model.add(MaxPooling2D())\n", "model.add(GlobalAveragePooling2D())\n", "model.add(Dense(5270, activation=\"softmax\"))\n", "\n", "model.compile(optimizer=\"adam\",\n", "              loss=\"categorical_crossentropy\",\n", "              metrics=[\"accuracy\"])\n", "\n", "#create the generators:\n", "\n", "train_img_gen = ImageDataGenerator() #Configure as you want\n", "train_gen = BinFileIterator('train.bin', img_generator=train_img_gen,  samples=1000,\n", "                 target_size=(180,180), \n", "                 batch_size=100)  \n", "\n", "val_img_gen = ImageDataGenerator() #Configure as you want\n", "val_gen = BinFileIterator('val.bin', img_generator=val_img_gen,  samples=1000,\n", "                 target_size=(180,180), \n", "                 batch_size=100) \n", "\n", "# To train the model:\n", "model.fit_generator(train_gen,\n", "                    steps_per_epoch = 1000/100,   #num_train_images // batch_size,\n", "                    epochs = 2,\n", "                    validation_data = val_gen,\n", "                    validation_steps = 1000/100,  #num_val_images // batch_size,\n", "                    workers = 4)"]}, {"metadata": {"_cell_guid": "ce4136ec-fe41-4399-9f37-0e2f40bf2e18", "_uuid": "abe3b4aac98ef14febac3d33312a07ccc2799f26"}, "source": ["I hope this kerner be usefull. If so, please upvote!"], "cell_type": "markdown"}], "nbformat_minor": 1, "nbformat": 4}