{"nbformat_minor": 1, "nbformat": 4, "metadata": {"language_info": {"mimetype": "text/x-python", "file_extension": ".py", "codemirror_mode": {"version": 3, "name": "ipython"}, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "version": "3.6.1", "name": "python"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}}, "cells": [{"execution_count": null, "outputs": [], "source": ["import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import bson,io\n", "from random import randint\n", "from skimage.data import imread"], "metadata": {"collapsed": true, "_uuid": "6a36cc5fd34f71a37eb17d50cb0af63a758bc4c1", "_cell_guid": "9c52fc39-1a5e-442f-bae2-3910a12a224e"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["data = bson.decode_file_iter(open('../input/train.bson', 'rb'))\n", "'''\n", "    data generator for keras fit_generator function\n", "'''\n", "\n", "def convert2onehot(category_id):\n", "    # you should have a dictionary for mapping each id to an index number\n", "    # for demonstration purposes, I will only include a dummy file\n", "    return True,category_id\n", "\n", "def data_generator(batch_size=128, start_image=0):\n", "    count_product = 0\n", "    images = []\n", "    y_label = []\n", "    while True:\n", "        count = 0\n", "        for c, d in enumerate(data):\n", "            category_id = d['category_id']\n", "            if count_product < start_image:\n", "                count_product += 1\n", "                continue\n", "            success, one_hot = convert2onehot(category_id) # be sure to create your own y_label output\n", "            if not success:\n", "                print(\"id conversion failed\")\n", "                continue\n", "            for e, pic in enumerate(d['imgs']):\n", "                picture = imread(io.BytesIO(pic['picture']))\n", "                images.append(picture)\n", "                y_label.append(one_hot)\n", "                count += 1\n", "            if count >= batch_size:\n", "                count = 0\n", "                y_label = np.asarray(y_label)\n", "                images = np.asarray(images)\n", "                '''\n", "                    since shuffle in fit function will not work here, \n", "                    a batch shuffle mechnism is added \n", "                '''\n", "                for i,image in enumerate(images[:int(batch_size/2)]):\n", "                    j = randint(0,batch_size-1)\n", "                    y_temp = y_label[i]\n", "                    img_temp = image\n", "                    images[i] = images[j]\n", "                    y_label[i] = y_label[j]\n", "                    images[j] = img_temp\n", "                    y_label[j] = y_temp\n", "                yield images, y_label\n", "                # just to be sure past batch are removed from the memory\n", "                del images\n", "                del y_label\n", "                images = []\n", "                y_label = []\n", "                \n", "                \n", "'''\n", "    Example: \n", "    models.fit_generator(data_generator(BATCH_SIZE),steps_per_epoch=70000,nb_epoch=NB_EPOCH,callbacks=[validate_callback],workers=1)\n", "    Note: steps per epoch is the total iteration in one epoch\n", "'''"], "metadata": {"_uuid": "819146f8f8c1de467eec861b40cf3a0c3849800e", "_cell_guid": "4e3220ac-75d8-4279-9e89-5b765d191b4d"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["train_gen = data_generator()\n", "next(train_gen)  # warm-up\n", "\n", "%time bx, by = next(train_gen)"], "metadata": {}, "cell_type": "code"}, {"source": ["### Performance evaluation\n", "#### 104 ms need for one single batch\n", "\n", "### Keras Validation Callback\n", "   since fit_generator won't show any other metrics other than loss,\n", "   and for some reason my fit_generator cannot accept validation data parameters,\n", "   I also implemented a validation callbacks for validating the accuracy on my \n", "   validation dataset\n"], "metadata": {}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "source": ["\n", "import pickle\n", "from keras.callbacks import Callback\n", "\n", "class ValidateCallbacks(Callback):\n", "    # validation data is preprocessed and saved as pickle file\n", "    def __init__(self):\n", "        self.test_data = pickle.load(open(\"data/validation_data.p\",\"rb\"))\n", "\n", "    def on_epoch_end(self, epoch, logs=None):\n", "        images,y_label = self.test_data\n", "        y_pred = self.model.predict(images)\n", "        batch_size = len(y_pred)\n", "        count = 0\n", "        for index, pred in enumerate(y_pred):\n", "            y_id, max_prob = get_max(pred)\n", "            if y_id == y_label[index]:\n", "                count += 1\n", "        print(\"  Validation Score {}\".format((count * 1.0) / (batch_size * 1.0)))"], "metadata": {"_uuid": "743c1a882f1f077d99a23dce8a93ba0ce429636d", "_cell_guid": "46050c96-7544-484b-af42-087ef09f51c2"}, "cell_type": "code"}, {"source": ["\n", "# Happy training\n", "#### If you find this useful please give me a thumbs up\n"], "metadata": {"_uuid": "a58141c484ff6228186e47b9ebf9357e874eeb76", "_cell_guid": "fb2d0a76-8f6a-4ce4-b037-8906edc82821"}, "cell_type": "markdown"}]}