{"cells": [{"cell_type": "markdown", "metadata": {"_uuid": "85bde2dd7e71ccf5e46edfacf722877be8b721c9", "_cell_guid": "13a22894-2f0e-43ff-a642-decbcd015d1d"}, "source": ["# Data visualization and classification algorithms\n", "\n", "Hello everyone, here is my notebook for the CDiscount challenge. \n", "\n", "I'm greatly thankful to vfdev for his incredible notebook https://www.kaggle.com/vfdev5/data-visualization-and-analysis that inspired this one."]}, {"outputs": [], "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import psutil #useful to see memory usage\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "cell_type": "code", "metadata": {"_uuid": "782d8fdf0e02db84563440b3fbb32f548566e044", "_cell_guid": "2aac43a5-a335-41f1-ac4f-22c102cec4f2"}, "execution_count": 1}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "48312e8c6d597dcb1f16749b84c972718f45b2f2", "_cell_guid": "6b0c5314-b040-4297-9687-fc264e31147a"}, "execution_count": 2}, {"outputs": [], "source": ["import io\n", "import os\n", "import bson\n", "import matplotlib.pyplot as plt\n", "\n", "INPUT_PATH = os.path.join('..', 'input')\n", "CATEGORY_NAMES_DF = pd.read_csv(os.path.join(INPUT_PATH, 'category_names.csv'))\n", "TRAIN_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'train.bson'), 'rb'))\n", "TRAIN_EXAMPLE_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'train_example.bson'), 'rb'))\n", "TEST_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'test.bson'), 'rb'))"], "cell_type": "code", "metadata": {"_uuid": "8840ffd966c04f3ee22469e899beedf5d058c6e7", "collapsed": true, "_cell_guid": "9022032a-43c1-4042-8969-d07a9dc4238d"}, "execution_count": 3}, {"outputs": [], "source": ["CATEGORY_NAMES_DF.head(5)"], "cell_type": "code", "metadata": {"_uuid": "b197dcb4ba89c2616720be3b060684ebd15dcbcc", "_cell_guid": "1b07bfa5-8246-4ed8-87b5-136d9b19cd4d"}, "execution_count": 4}, {"outputs": [], "source": ["CAT = pd.DataFrame(CATEGORY_NAMES_DF.category_id)\n", "CAT['category_nb'] = CAT.index\n", "CATEGORY_NAMES_DF = pd.merge(CAT, CATEGORY_NAMES_DF, on = ['category_id'])\n", "CATEGORY_NAMES_DF.head()"], "cell_type": "code", "metadata": {"_uuid": "cb515b686f7aabd624b7991b674d7fcdc81bd2c9", "_cell_guid": "bad1b32a-b54d-459c-9e59-4c4a0a49c277"}, "execution_count": 5}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "6b2ac74c8101a83c131f7606d194c18981ef210f", "_cell_guid": "a4f6e073-4542-43c8-a45b-8a051c4143cc"}, "execution_count": 6}, {"cell_type": "markdown", "metadata": {"_uuid": "c7861c2fb359abcf595ebaafc621bbf9a45dcea3", "_cell_guid": "6c08aa1a-0528-48e6-ab75-e95d918e2487"}, "source": ["# First Images of train and test set"]}, {"outputs": [], "source": ["for item in TRAIN_DB:\n", "    break\n", "print(type(item))\n", "print(item.keys())\n", "print(item['_id'], item['category_id'], type(item['imgs']), len(item['imgs']))"], "cell_type": "code", "metadata": {"_uuid": "e21e932eda2932012c35c66ac7775f1da63f5310", "_cell_guid": "d58ec8a1-5668-4610-8b5d-7661367349d1"}, "execution_count": 7}, {"outputs": [], "source": ["import cv2\n", "from PIL import Image\n", "\n", "def decode_image(data):\n", "    arr = np.asarray(bytearray(data), dtype=np.uint8)\n", "    img = cv2.imdecode(arr, cv2.IMREAD_COLOR)\n", "    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n", "\n", "def decode_pil(data):\n", "    return Image.open(io.BytesIO(data))\n", "\n", "for img_dict in item['imgs']:\n", "    img = decode_image(img_dict['picture'])\n", "    plt.figure()\n", "    plt.imshow(img)\n", "    plt.grid(False)\n", "    \n", "#Alternatively\n", "#for e, pic in enumerate(item['imgs']):\n", "#    picture = imread(io.BytesIO(pic['picture']))\n", "#    plt.imshow(picture)\n", "#    plt.show()"], "cell_type": "code", "metadata": {"_uuid": "4e89756d93ea4045a0b27a8aff771aa9247967b6", "_cell_guid": "b838b528-a35e-45c6-8f3b-59ccd709374b"}, "execution_count": 8}, {"outputs": [], "source": ["level_tags = CATEGORY_NAMES_DF.columns[2:]\n", "CATEGORY_NAMES_DF[CATEGORY_NAMES_DF['category_id'] == item['category_id']][level_tags]"], "cell_type": "code", "metadata": {"_uuid": "c8d759f1a5e5c1af2919564bd677a9766098f451", "_cell_guid": "7edc5b90-a9a4-4486-aa30-b04da4b36e55"}, "execution_count": 9}, {"cell_type": "markdown", "metadata": {"_uuid": "485411394af8dffbd2bf61c466be1609a49ca67f", "_cell_guid": "a2a4debf-8bd0-4126-8ca5-96a1d4121786"}, "source": ["Let's show some more images"]}, {"outputs": [], "source": ["def decode_images(item_imgs):\n", "    nb_imgs = len(item_imgs)\n", "    nx = 2 if nb_imgs > 1 else 1\n", "    ny = 2 if nb_imgs > 2 else 1\n", "    set_imgs = np.zeros((180*ny, 180*nx,3), dtype = np.uint8)\n", "    for i,img_dict in enumerate(item_imgs):\n", "        img = decode_image(img_dict['picture'])\n", "        h, w, _ = img.shape        \n", "        xstart = (i % nx) * 180\n", "        xend = xstart + w\n", "        ystart = (i // nx) * 180\n", "        yend = ystart + h\n", "        set_imgs[ystart:yend, xstart:xend] = img\n", "    return set_imgs"], "cell_type": "code", "metadata": {"_uuid": "5d750e6fee979bfa8585a5d019554d088f1009cf", "collapsed": true, "_cell_guid": "39e1c089-3bb5-41cb-bded-536fc4e1e205"}, "execution_count": 10}, {"outputs": [], "source": ["#Reset the iterator\n", "TRAIN_EXAMPLE_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'train_example.bson'), 'rb'))\n", "prod_to_category = dict()\n", "k = 0\n", "\n", "rand_rows = np.random.permutation(82)\n", "fig, axArray = plt.subplots(nrows=2,ncols=2, figsize=(16,8))\n", "plt.subplots_adjust(wspace=0.1, hspace=0.6)\n", "for c, d in enumerate(TRAIN_EXAMPLE_DB):\n", "    product_id = d['_id']\n", "    category_id = d['category_id']\n", "    prod_to_category[product_id] = category_id\n", "    picture = decode_images(d['imgs'])\n", "    if(c in rand_rows[0:4]):\n", "        mask = CATEGORY_NAMES_DF['category_id'] == d['category_id']\n", "        cat_levels = CATEGORY_NAMES_DF[mask][level_tags].values.tolist()[0]\n", "        cat_levels = [c[:25] for c in cat_levels]\n", "        title = str(d['category_id']) + '\\n'\n", "        title += '\\n'.join(cat_levels)\n", "        nx = 1 if k % 2 == 0 else 0\n", "        ny = 1 if k // 2 == 0 else 0\n", "        k += 1\n", "        axArray[ny][nx].set_title(title)\n", "        axArray[ny][nx].imshow(picture)\n", "plt.show()\n", "\n", "prod_to_category = pd.DataFrame.from_dict(prod_to_category, orient='index')\n", "prod_to_category.index.name = '_id'\n", "prod_to_category.rename(columns={0: 'category_id'}, inplace=True)"], "cell_type": "code", "metadata": {"_uuid": "31c405442b430122556e4577b6a1bb7b43f47a8f", "_cell_guid": "3721cb85-f3c9-4538-a5d2-9ff76f006e3b"}, "execution_count": 11}, {"outputs": [], "source": ["prod_to_category.head(5)"], "cell_type": "code", "metadata": {"_uuid": "940eb67fca44adc623ee55adf0d12224981e405e", "_cell_guid": "78b77e25-7f48-4f3a-b511-e49694a863ac"}, "execution_count": 12}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "143f813281c9919413adf878fb778cdeab5be64a", "_cell_guid": "a889b6b7-0126-4d5a-b9f3-9c28a6e0f358"}, "execution_count": 13}, {"cell_type": "markdown", "metadata": {"_uuid": "ba12526838faced5251b549974d254553be1aba5", "_cell_guid": "036a28d9-970d-47f8-abdf-305330012061"}, "source": ["# The test set"]}, {"outputs": [], "source": ["#Reset the iterator\n", "TEST_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'test.bson'), 'rb'))\n", "n = 4\n", "\n", "maxcounter = 15\n", "for c, item in enumerate(TEST_DB):\n", "    if c % n == 0:\n", "        plt.figure(figsize=(14,6))\n", "    \n", "    plt.subplot(1, n, c % n + 1)\n", "    title = str(item['_id'])\n", "    plt.imshow(decode_images(item['imgs']))\n", "    plt.title(title)\n", "    plt.axis('off')\n", "    \n", "    if c==maxcounter:\n", "        break"], "cell_type": "code", "metadata": {"_uuid": "79ed8c6966d788923a923c8feed888876aef5e5a", "_cell_guid": "fc7d86c5-c538-4fa3-add3-7ae5e78119f9"}, "execution_count": 14}, {"cell_type": "markdown", "metadata": {"_uuid": "bbc5c7feda5ff7770845ce59b21f7d9e8aab19b4", "_cell_guid": "7860c141-8767-4139-9024-75c77d15bdd1"}, "source": ["Following code creates a dictionary with key indexing item _id and values (offset, length). It takes around 3 mins to execute."]}, {"outputs": [], "source": ["import struct\n", "from tqdm import tqdm_notebook\n", "\n", "num_dicts = 7069896 # according to data page\n", "length_size = 4\n", "IDS_MAPPING = {}\n", "\n", "with open(os.path.join(INPUT_PATH, 'train.bson'), 'rb') as f, tqdm_notebook(total=num_dicts) as bar:\n", "    item_data = []\n", "    offset = 0\n", "    while True:        \n", "        bar.update()\n", "        f.seek(offset)\n", "        \n", "        item_length_bytes = f.read(length_size)     \n", "        if len(item_length_bytes) == 0:\n", "            break                \n", "        # Decode item length:\n", "        length = struct.unpack(\"<i\", item_length_bytes)[0]\n", "        \n", "        f.seek(offset)\n", "        item_data = f.read(length)\n", "        assert len(item_data) == length, \"%i vs %i\" % (len(item_data), length)\n", "        \n", "        # Check if we can decode\n", "        item = bson.BSON.decode(item_data)\n", "        \n", "        IDS_MAPPING[item['_id']] = (offset, length)        \n", "        offset += length            \n", "            \n", "def get_item(item_id):\n", "    assert item_id in IDS_MAPPING\n", "    with open(os.path.join(INPUT_PATH, 'train.bson'), 'rb') as f:\n", "        offset, length = IDS_MAPPING[item_id]\n", "        f.seek(offset)\n", "        item_data = f.read(length)\n", "        return bson.BSON.decode(item_data)"], "cell_type": "code", "metadata": {"_uuid": "f646fd095310d9ae9edc01e9168da93ce61d9a75", "_cell_guid": "7284d703-3327-4c9e-940d-3b84d138594a"}, "execution_count": 15}, {"outputs": [], "source": ["print(psutil.virtual_memory())\n", "print(psutil.cpu_times())"], "cell_type": "code", "metadata": {"_uuid": "1608fcafc86057f9e88d882fc3ec306181e22deb", "_cell_guid": "1a465219-8e46-4691-9bb6-e1bdd85807c5"}, "execution_count": 16}, {"cell_type": "markdown", "metadata": {"_uuid": "200c45961b960492577e245e63d3892fc1e6fddb", "_cell_guid": "53221d11-6c30-47d1-99a3-c5629cfb142e"}, "source": ["Let's display for example item with _id 1234 "]}, {"outputs": [], "source": ["item = get_item(1234)\n", "\n", "mask = CATEGORY_NAMES_DF['category_id'] == item['category_id']\n", "cat_levels = CATEGORY_NAMES_DF[mask][level_tags].values.tolist()[0]\n", "cat_levels = [c[:25] for c in cat_levels]\n", "title = str(item['category_id']) + '\\n'\n", "title += '\\n'.join(cat_levels)\n", "plt.imshow(decode_images(item['imgs']))\n", "plt.title(title)\n", "plt.axis('off')\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "d23393a70ae493373d2b59629f81039138cc2551", "_cell_guid": "e1f00633-2f21-44aa-8cbf-17a2f2f65064"}, "execution_count": 17}, {"cell_type": "markdown", "metadata": {"_uuid": "35b1f216393f101d49b3db4ca98e04a7b75786f7", "_cell_guid": "ba83c19f-1f3c-48f3-b770-4113eb011b02"}, "source": ["# Explore the categories\n", "We have have three levels of categories\n", "*  49 unique level 1 categories\n", "* 483 unique level 2 categories\n", "* 5263 unique level 3 categories"]}, {"outputs": [], "source": ["print(\"Number of categories %i\"% CATEGORY_NAMES_DF.category_id.nunique())\n", "print(\"Number of level 1 categories %i\"% CATEGORY_NAMES_DF.category_level1.nunique())\n", "print(\"Number of level 2 categories %i\"% CATEGORY_NAMES_DF.category_level2.nunique())\n", "print(\"Number of level 3 categories %i\"% CATEGORY_NAMES_DF.category_level3.nunique())"], "cell_type": "code", "metadata": {"_uuid": "2cd80cffd100c8d04a7535625ac6d6fe3527871d", "_cell_guid": "a7f5b53e-6326-4928-80cb-58b950521a14"}, "execution_count": null}, {"outputs": [], "source": ["import seaborn as sns\n", "\n", "#Histogram of level1 categories\n", "plt.figure(figsize=(12,12))\n", "sns.countplot(y=CATEGORY_NAMES_DF.category_level1)\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "390694f09de644590536bc7266e28a1557139de3", "collapsed": true, "_cell_guid": "9c5fbeef-3da0-462f-b1c4-8790a65e800b"}, "execution_count": null}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "32766c0e340ef5f9f078c9ee733f80e772f42cfa", "collapsed": true, "_cell_guid": "3fec1999-f973-412c-b1cc-abf706d0c187"}, "execution_count": null}, {"cell_type": "markdown", "metadata": {"_uuid": "7e83d8ca78c862735194f5d9cb431bb1306f0cf4", "_cell_guid": "2ba3b541-7994-410a-a41e-a549845c95f0"}, "source": ["# Explore the training set \n", "\n", "Now let's create the training table _id, category_id"]}, {"outputs": [], "source": ["num_dicts = 7069896\n", "prod_to_category = [None] * num_dicts\n", "TRAIN_DB = bson.decode_file_iter(open(os.path.join(INPUT_PATH, 'train.bson'), 'rb'))\n", "\n", "with tqdm_notebook(total=num_dicts) as bar:\n", "    for i, item in enumerate(TRAIN_DB):\n", "        bar.update()\n", "        prod_to_category[i] = (item['_id'],item['category_id'])"], "cell_type": "code", "metadata": {"_uuid": "71103d5f0368e58ebc6741948939025ba71bf526", "collapsed": true, "_cell_guid": "1f570c3c-bc5b-45eb-9b54-27c9890ec99c"}, "execution_count": null}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "664a4e77477d737a6bc5857e29252b94c044d05a", "collapsed": true, "_cell_guid": "79220b05-5143-43b9-9647-edcf502568a8"}, "execution_count": null}, {"outputs": [], "source": ["TRAIN_CATEGORIES_DF = pd.DataFrame(prod_to_category, columns=['_id', 'category_id'])\n", "TRAIN_CATEGORIES_DF.head()"], "cell_type": "code", "metadata": {"_uuid": "c59a5d3dadbef701b0f4ca4fcc52e6bc1a2583dd", "collapsed": true, "_cell_guid": "f7b48571-1049-4d8c-a24b-16eabbbbcb84"}, "execution_count": null}, {"outputs": [], "source": ["TRAIN_DF = pd.merge(TRAIN_CATEGORIES_DF, CATEGORY_NAMES_DF, on = ['category_id'])"], "cell_type": "code", "metadata": {"_uuid": "2ca218ad613c61306e15d95d6b71a62dafb8ec02", "collapsed": true, "_cell_guid": "385ac5a8-96b7-485c-bd90-7c40d0e63ddc"}, "execution_count": null}, {"outputs": [], "source": ["TRAIN_DF.head(5)"], "cell_type": "code", "metadata": {"_uuid": "0217401138db79a3b08580c93fbe6d7afe460e1a", "collapsed": true, "_cell_guid": "f03c42bf-a81e-4057-9b7d-ac1758a6c1e4"}, "execution_count": null}, {"cell_type": "markdown", "metadata": {"_uuid": "9b429e1eb994f81ec224a507cc912458d7228245", "_cell_guid": "e6e2d24f-5efb-4200-9de4-817225146fed"}, "source": ["Little double check to make sure we didn't lose some products in the merge"]}, {"outputs": [], "source": ["TRAIN_DF._id.unique().sort() == TRAIN_CATEGORIES_DF._id.unique().sort()"], "cell_type": "code", "metadata": {"_uuid": "37783c22183c63f80ff212f8e4f6376949c99e4b", "collapsed": true, "_cell_guid": "80ce1818-855b-4eff-8146-376b309af722"}, "execution_count": null}, {"outputs": [], "source": ["#Histogram of level1 categories\n", "plt.figure(figsize=(12,12))\n", "sns.countplot(y=TRAIN_DF.category_level1)\n", "plt.title(\"Train set level1 histogram\")\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "51a7249bcb40bd1b30e7601a043750e53b93c553", "collapsed": true, "_cell_guid": "93d3a9f4-f6a7-4e4f-b4d3-67b0894bc319"}, "execution_count": null}, {"outputs": [], "source": ["train_gb = TRAIN_DF.groupby('category_id')\n", "train_count = train_gb['category_id'].count()\n", "\n", "most_freq_cats = train_count[train_count == train_count.max()]\n", "print(\"Most frequent category: \", CATEGORY_NAMES_DF[CATEGORY_NAMES_DF['category_id'].isin(most_freq_cats.index)].values)"], "cell_type": "code", "metadata": {"_uuid": "3ffb3c7561b7260487127c4547d89effa8d98e05", "collapsed": true, "_cell_guid": "57f35d16-f6bb-4069-a74b-48f15f1d4fec"}, "execution_count": null}, {"outputs": [], "source": ["most_freq_cat = most_freq_cats.index[0]\n", "TRAIN_MOST_FREQ_DF = TRAIN_DF[TRAIN_DF['category_id']==most_freq_cat]\n", "\n", "mask = CATEGORY_NAMES_DF['category_id'] == most_freq_cat   \n", "cat_levels = CATEGORY_NAMES_DF[mask][level_tags].values.tolist()[0]\n", "title = str(item['category_id']) + '\\n'\n", "title += '\\n'.join(cat_levels)\n", "\n", "maxcounter = 50\n", "n = 10\n", "c = 0\n", "for item_id in TRAIN_MOST_FREQ_DF['_id'][:maxcounter]:\n", "    if c % n == 0:\n", "        plt.figure(figsize=(14,4))\n", "        if c == 0:\n", "            plt.suptitle(title)\n", "    \n", "    item = get_item(item_id)\n", "    plt.subplot(1, n, c % n + 1)\n", "    plt.imshow(decode_images(item['imgs']))\n", "    plt.axis('off')\n", "    \n", "    c += 1\n", "    if c==maxcounter:\n", "        break\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "1af1c6ac2166f94c0697589c0273cd87d1ed0864", "collapsed": true, "_cell_guid": "d3db79b6-b541-43d5-8b23-5f8b1814c2be"}, "execution_count": null}, {"outputs": [], "source": ["item = get_item(1234)\n", "img = decode_images(item['imgs'])\n", "plt.imshow(img)\n", "plt.axis('off')\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "dffbc52f302d337f98b6ffa500064999f677bdab", "collapsed": true, "_cell_guid": "de2c91fc-eada-4e8c-94ef-283f7e6c0a0c"}, "execution_count": null}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "9a7fa85c7a7d0eb1e64d669b7acd50d93d713db4", "collapsed": true, "_cell_guid": "2102dfe3-0ab5-4b17-91d4-a2cd9d77e8c8"}, "execution_count": null}, {"cell_type": "markdown", "metadata": {"_uuid": "f681fca88b344d487861ddfea3b08493779ec429", "collapsed": true, "_cell_guid": "c9d879b0-7cea-4f3d-8882-2485545c14f6"}, "source": ["# Prediction Model"]}, {"outputs": [], "source": ["import theano\n", "from theano import tensor as T\n", "from theano.tensor.nnet import conv2d\n", "\n", "rng = np.random.RandomState(23455)\n", "\n", "#instantiate 4D tensor for input\n", "input = T.tensor4(name='input')\n", "\n", "#initialize shared variables for weights\n", "w_shp = (4,3,9,9)\n", "w_bound = np.sqrt(3*9*9)\n", "W = theano.shared(np.asarray(rng.uniform(low=-1.0/w_bound, high=1.0/w_bound, size=w_shp),\n", "                            dtype = input.dtype), name='W')\n", "\n", "# IMPORTANT: biases are usually initialized to zero. However in this\n", "# particular application, we simply apply the convolutional layer to\n", "# an image without learning the parameters. We therefore initialize\n", "# them to random values to \"simulate\" learning.\n", "b_shp = (4,)\n", "b = theano.shared(np.asarray(\n", "            rng.uniform(low=-.5, high=.5, size=b_shp),\n", "            dtype=input.dtype), name ='b')\n", "\n", "# build symbolic expression that computes the convolution of input with filters in w\n", "conv_out = conv2d(input, W)\n", "output = T.nnet.sigmoid(conv_out + b.dimshuffle('x', 0, 'x', 'x'))\n", "# create theano function to compute filtered images\n", "f = theano.function([input], output)"], "cell_type": "code", "metadata": {"_uuid": "05aa3c7ef3fd583f3b00ca4769e18a628c3a589b", "collapsed": true, "_cell_guid": "9232d355-8dde-4970-b5a8-9905e7410012"}, "execution_count": null}, {"outputs": [], "source": ["from PIL import Image\n", "\n", "item = get_item(1)\n", "img = decode_images(item['imgs'])\n", "img = img/256.\n", "\n", "# put image in 4D tensor of shape (1, 3, height, width)\n", "img_ = img.transpose(2, 0, 1).reshape(1, 3, 180,180)\n", "filtered_img = f(img_)\n", "# plot original image and first and second components of output)\n", "plt.subplot(1, 5, 1); plt.axis('off'); plt.imshow(img)\n", "plt.gray();\n", "# recall that the convOp output (filtered image) is actually a \"minibatch\",\n", "# of size 1 here, so we take index 0 in the first dimension:\n", "plt.subplot(1, 5, 2); plt.axis('off'); plt.imshow(filtered_img[0, 0, :, :])\n", "plt.subplot(1, 5, 3); plt.axis('off'); plt.imshow(filtered_img[0, 1, :, :])\n", "plt.subplot(1, 5, 4); plt.axis('off'); plt.imshow(filtered_img[0, 2, :, :])\n", "plt.subplot(1, 5, 5); plt.axis('off'); plt.imshow(filtered_img[0, 3, :, :])\n", "plt.show()"], "cell_type": "code", "metadata": {"_uuid": "da4a2f7f6a107094f16e65d50ff090c019963615", "collapsed": true, "_cell_guid": "69a60a4e-83ef-4293-a23e-5e9045608817"}, "execution_count": null}, {"outputs": [], "source": ["from theano.tensor.signal import pool\n", "\n", "input = T.dtensor4('input')\n", "maxpool_shape = (2,2)\n", "pool_out = pool.pool_2d(input, maxpool_shape, ignore_border=True)\n", "f = theano.function([input], pool_out)\n", "\n", "img_pool = f(img_)\n", "plt.subplot(1,2,1) ; plt.axis('off') ; plt.imshow(img)\n", "plt.subplot(1,2,2) ; plt.axis('off') ; plt.imshow(img_pool[0].transpose(1,2,0))"], "cell_type": "code", "metadata": {"_uuid": "68d456c46b1d9d0d720540fa7ee3261d25541310", "collapsed": true, "_cell_guid": "15ddd08d-4790-4190-82a8-d60680878de0"}, "execution_count": null}, {"outputs": [], "source": ["import six.moves.cPickle as pickle\n", "import gzip\n", "import os\n", "import sys\n", "import timeit\n", "\n", "\n", "class LogisticRegression(object):\n", "    def __init__(self, input, n_in, n_out):\n", "        self.W = theano.shared(value=np.zeros((n_in, n_out), dtype = theano.config.floatX),\n", "                               name = 'W', borrow=True)\n", "        self.b = theano.shared(value=np.zeros((n_out,), dtype = theano.config.floatX),\n", "                               name = 'b', borrow=True)\n", "        self.p_y_given_x = T.nnet.softmax(T.dot(input,self.W)+self.b)\n", "        self.y_pred = T.argmax(self.p_y_given_x, axis=1)\n", "        self.params = [self.W, self.b]\n", "        self.input = input\n", "        \n", "    def negative_log_likelihood(self,y):\n", "        return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y])\n", "    \n", "    def errors(self,y):\n", "        if y.ndim != self.y_pred.ndim:\n", "            raise TypeError(\n", "                'y should have the same shape as self.y_pred',\n", "                ('y', y.type, 'y_pred', self.y_pred.type)\n", "            )\n", "\n", "        if y.dtype.startswith('int'):\n", "            #1 represents a mistake in prediction\n", "            return T.mean(T.neq(self.y_pred, y))\n", "        else:\n", "            raise NotImplementedError()\n", "            \n", "class HiddenLayer(object):\n", "    def __init__(self, rng, input, n_in, n_out, W=None, b=None,\n", "                 activation=T.tanh):\n", "        self.input = input\n", "        \n", "        if W is None:\n", "            W_values = np.asarray(\n", "                rng.uniform(\n", "                    low=-np.sqrt(6. / (n_in + n_out)),\n", "                    high=np.sqrt(6. / (n_in + n_out)),\n", "                    size=(n_in, n_out)\n", "                ),\n", "                dtype=theano.config.floatX\n", "            )\n", "            if activation == theano.tensor.nnet.sigmoid:\n", "                W_values *= 4\n", "\n", "            W = theano.shared(value=W_values, name='W', borrow=True)\n", "\n", "        if b is None:\n", "            b_values = np.zeros((n_out,), dtype=theano.config.floatX)\n", "            b = theano.shared(value=b_values, name='b', borrow=True)\n", "\n", "        self.W = W\n", "        self.b = b\n", "        \n", "        lin_output = T.dot(input, self.W) + self.b\n", "        if activation is None:\n", "            output = lin_output\n", "        else:\n", "            output = activation(lin_output)\n", "          \n", "        self.params = [self.W, self.b]\n", "        self.output = output\n"], "cell_type": "code", "metadata": {"_uuid": "e02d198870207289407460ca00c0722c188c2197", "collapsed": true, "_cell_guid": "8a14ba72-874f-436a-b7d0-aa4c85a30c8a"}, "execution_count": null}, {"outputs": [], "source": ["class LeNetConvPoolLayer(object):\n", "    \"\"\"Pool Layer of a convolutional network \"\"\"\n", "\n", "    def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):\n", "\n", "        assert image_shape[1] == filter_shape[1]\n", "        self.input = input\n", "        \n", "        fan_in = np.prod(filter_shape[1:])\n", "        fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) //\n", "                   np.prod(poolsize))\n", "        # initialize weights with random weights\n", "        W_bound = np.sqrt(6. / (fan_in + fan_out))\n", "        self.W = theano.shared(\n", "            np.asarray(\n", "                rng.uniform(low=-W_bound, high=W_bound, size=filter_shape),\n", "                dtype=theano.config.floatX\n", "            ),\n", "            borrow=True\n", "        )\n", "\n", "        b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX)\n", "        self.b = theano.shared(value=b_values, borrow=True)\n", "        \n", "        # convolve input feature maps with filters\n", "        conv_out = conv2d(\n", "            input=input,\n", "            filters=self.W,\n", "            filter_shape=filter_shape,\n", "            input_shape=image_shape\n", "        )\n", "\n", "        # pool each feature map individually, using maxpooling\n", "        pooled_out = pool.pool_2d(\n", "            input=conv_out,\n", "            ds=poolsize,\n", "            ignore_border=True\n", "        )\n", "\n", "        self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))\n", "        self.params = [self.W, self.b]\n", "        self.input = input"], "cell_type": "code", "metadata": {"_uuid": "05225dc0c9ff9dca9a9e02196e44c7004978b1e8", "collapsed": true, "_cell_guid": "fb7820cd-6c5e-4600-b53d-127441b4d379"}, "execution_count": null}, {"outputs": [], "source": ["class CNN(object):\n", "    def __init__(self,rng,input,batch_size,n_out, n_kerns, n_hidden):    \n", "        \n", "        # filter_shape is (n_output_channels,n_input_channels, filter_height, filter_width)\n", "        # filtering reduces the image size to (180-5+1 , 180-5+1) = (176, 176)\n", "        # poolsize = (2,2)  reduces it further to (176/2,176/2) = (88,88)\n", "        \n", "        \n", "        self.layer0=LeNetConvPoolLayer(\n", "                rng=rng,\n", "                input=input.reshape((batch_size,3,180,180)),\n", "                image_shape=(batch_size, 3, 180, 180),\n", "                filter_shape=(n_kerns[0], 3, 5, 5),\n", "                poolsize=(2, 2)\n", "        )\n", "        \n", "        # Construct the second convolutional pooling layer\n", "        # filtering reduces the image size to (88-5+1, 88-5+1) = (84,84)\n", "        # maxpooling reduces this further to (84/2, 84/2) = (42, 42)\n", "        # 4D output tensor is thus of shape (batch_size, 1, 42, 42)\n", "        self.layer1 = LeNetConvPoolLayer(\n", "            rng,\n", "            input=self.layer0.output,\n", "            image_shape=(batch_size, n_kerns[0], 88, 88),\n", "            filter_shape=(n_kerns[1], n_kerns[0], 5, 5),\n", "            poolsize=(2, 2)\n", "        )\n", "        \n", "        # filter_shape is (n_output_channels,n_input_channels, filter_height, filter_width)\n", "        # filtering reduces the image size to (42-3+1 , 42-3+1) = (40, 40)\n", "        # poolsize = (2,2)  reduces it further to (40/2,40/2) = (20,20)\n", "        \n", "        \n", "        self.layer2=LeNetConvPoolLayer(\n", "                rng=rng,\n", "                input=input.reshape((batch_size,3,42,42)),\n", "                image_shape=(batch_size, 3, 42, 42),\n", "                filter_shape=(n_kerns[0], 3, 3, 3),\n", "                poolsize=(2, 2)\n", "        )\n", "        \n", "        \n", "        # the HiddenLayer being fully-connected, it operates on 2D matrices of\n", "        # shape (batch_size, num_pixels) (i.e matrix of rasterized images).\n", "        # This will generate a matrix of shape (batch_size, nkerns[1] * 20 * 20),\n", "        # or (batch_size, 3 * 20 * 20) = (batch_size, 1200) with the default values.\n", "        # construct a fully-connected sigmoidal layer\n", "        \n", "        self.layer3 = HiddenLayer(\n", "            rng,\n", "            input=self.layer2.output.flatten(2),\n", "            n_in=n_kerns[1] * 20 * 20,\n", "            n_out=n_hidden,\n", "            activation=T.tanh\n", "        )\n", "        \n", "        #n_out is the number of categories\n", "        self.layer4 = LogisticRegression(\n", "            input=self.layer3.output, \n", "            n_in=n_hidden, \n", "            n_out=n_out\n", "        )\n", "        \n", "        self.negative_log_likelihood = (\n", "            self.layer4.negative_log_likelihood\n", "        )\n", "\n", "        self.errors = self.layer4.errors\n", "        self.params = self.layer4.params + self.layer3.params + self.layer2.params \\\n", "            + self.layer1.params + self.layer0.params\n", "        self.input = input"], "cell_type": "code", "metadata": {"_uuid": "db92df16dc7926abf3d849392a6953f0d694cfe4", "collapsed": true, "_cell_guid": "1109e9a9-778a-4448-a45b-6ec89e6f4311"}, "execution_count": null}, {"outputs": [], "source": ["def load_dataset(rand_rows, offset, length):\n", "    \n", "    n_train = np.int(0.6*length)\n", "    n_valid = np.int(0.2*length)\n", "    n_test = np.int(0.2*length)\n", "     \n", "    train_set_x = np.zeros((n_train,3,180,180), dtype=float)\n", "    train_set_y = np.zeros((n_train,), dtype=float)\n", "        \n", "    valid_set_x = np.zeros((n_valid,3,180,180), dtype=float)\n", "    valid_set_y = np.zeros((n_valid,), dtype=float)\n", "    \n", "    test_set_x = np.zeros((n_test,3,180,180), dtype=float)\n", "    test_set_y = np.zeros((n_test,), dtype=float)\n", "       \n", "    #with tqdm_notebook(total=n_train) as bar:\n", "    for iter in range(offset,n_train+offset):\n", "        item = get_item(TRAIN_DF._id[rand_rows[iter]])\n", "        img = decode_images([item['imgs'][0]])\n", "        mask = CATEGORY_NAMES_DF['category_id'] == item['category_id']\n", "        train_set_x[iter-offset] = img.transpose(2, 0, 1).reshape(3, 180,180)\n", "        train_set_y[iter-offset] = CATEGORY_NAMES_DF[mask]['category_nb'].values.tolist()[0]\n", "    \n", "    #with tqdm_notebook(total=n_valid) as bar:\n", "    for iter in range(offset+n_train, offset+n_train + n_valid):\n", "        item = get_item(TRAIN_DF._id[rand_rows[iter]])\n", "        img = decode_images([item['imgs'][0]])\n", "        mask = CATEGORY_NAMES_DF['category_id'] == item['category_id']\n", "        valid_set_x[iter-n_train-offset] = img.transpose(2, 0, 1).reshape(3, 180,180)\n", "        valid_set_y[iter-n_train-offset] = CATEGORY_NAMES_DF[mask]['category_nb'].values.tolist()[0]\n", "    \n", "    #with tqdm_notebook(total=n_test) as bar:\n", "    for iter in range(offset+n_train+n_valid, offset+n_train + n_valid+n_test):\n", "        item = get_item(TRAIN_DF._id[rand_rows[iter]])\n", "        img = decode_images([item['imgs'][0]])\n", "        mask = CATEGORY_NAMES_DF['category_id'] == item['category_id']\n", "        test_set_x[iter-n_train-n_valid-offset] = img.transpose(2, 0, 1).reshape(3, 180,180)\n", "        test_set_y[iter-n_train-n_valid-offset] = CATEGORY_NAMES_DF[mask]['category_nb'].values.tolist()[0]\n", "\n", "    train_set = (train_set_x, train_set_y)\n", "    valid_set = (valid_set_x, valid_set_y)\n", "    test_set = (test_set_x, test_set_y)\n", "    \n", "    def shared_dataset(data_xy, borrow=True):\n", "        data_x,data_y = data_xy\n", "        \n", "        shared_x = theano.shared(np.asarray(data_x, dtype=theano.config.floatX), borrow=borrow)\n", "        shared_y = theano.shared(np.asarray(data_y, dtype=theano.config.floatX), borrow=borrow)\n", "        return shared_x, T.cast(shared_y, 'int32')\n", "\n", "    test_set_x, test_set_y = shared_dataset(test_set)\n", "    valid_set_x, valid_set_y = shared_dataset(valid_set)\n", "    train_set_x, train_set_y = shared_dataset(train_set)\n", "\n", "    rval = [(train_set_x, train_set_y), (valid_set_x, valid_set_y),\n", "            (test_set_x, test_set_y)]\n", "    return rval"], "cell_type": "code", "metadata": {"_kg_hide-input": true, "collapsed": true, "_uuid": "b203ac6a1cfc9ebb91e5388bbf36352a0bcd9529", "_cell_guid": "67681d44-499a-42a9-92e2-3a0f6c4392f1"}, "execution_count": null}, {"outputs": [], "source": ["psutil.virtual_memory()"], "cell_type": "code", "metadata": {"_uuid": "fce8f5cac247628bcb86a3d7cb2edf4ba158d10d", "collapsed": true, "_cell_guid": "40e5dae8-d2e0-42bc-9635-838348667968"}, "execution_count": null}, {"outputs": [], "source": ["def test_cnn(learning_rate = 0.01, L1_reg = 0.00, L2_reg = 0.0001, n_epochs = 100,\n", "             batch_size = 10000, mini_batch_size = 100, n_kerns=(3,3),\n", "             n_hidden=750, n_out = 5270):\n", "    \n", "    #batch_size is the total number of images loaded in memory\n", "    \n", "    n_train_batches = np.int((0.6*batch_size)//mini_batch_size)\n", "    n_valid_batches = np.int((0.2*batch_size)//mini_batch_size)\n", "    n_test_batches = np.int((0.2*batch_size)//mini_batch_size)\n", "\n", "\n", "    ######################\n", "    # BUILD ACTUAL MODEL #\n", "    ######################\n", "    print('... building the model')\n", "    \n", "    index = T.lscalar()\n", "    size = T.lscalar()\n", "    x = T.tensor4('x')\n", "    y = T.ivector('y')\n", "    \n", "    rng = np.random.RandomState(1234)\n", "    \n", "    classifier = CNN(rng=rng,\n", "                     input = x.reshape((mini_batch_size,3,180,180)),\n", "                     batch_size = mini_batch_size,\n", "                     n_out = n_out,\n", "                     n_kerns = n_kerns,\n", "                     n_hidden = n_hidden\n", "    )\n", "    \n", "    cost = classifier.negative_log_likelihood(y)\n", "    gparams = [T.grad(cost, param) for param in classifier.params]\n", "    updates = [(param, param-learning_rate*gparam) for param,gparam in zip(classifier.params, gparams)]\n", "    \n", "    print('... training')\n", "\n", "    # early-stopping parameters\n", "    patience = 1000  # look as this many examples regardless\n", "    patience_increase = 2  # wait this much longer when a new best is\n", "                           # found\n", "    improvement_threshold = 0.995  # a relative improvement of this much is\n", "                                   # considered significant\n", "    validation_frequency = min(n_train_batches, patience // 2)\n", "                                  # go through this many\n", "                                  # minibatch before checking the network\n", "                                  # on the validation set; in this case we\n", "                                  # check every epoch\n", "\n", "    best_validation_loss = np.inf\n", "    best_iter = 0\n", "    test_score = 0.\n", "    start_time = timeit.default_timer()\n", "\n", "    epoch = 0\n", "    done_looping = False\n", "    \n", "    rand_rows = np.random.permutation(7069896)\n", "    offset = 0\n", "\n", "    while (epoch < n_epochs) and (not done_looping):\n", "        epoch = epoch + 1\n", "\n", "        dataset = load_dataset(rand_rows=rand_rows, offset=0, length=batch_size)\n", "        \n", "        train_set_x, train_set_y = dataset[0]\n", "        valid_set_x, valid_set_y = dataset[1]\n", "        test_set_x, test_set_y = dataset[2]\n", "        \n", "        test_model = theano.function(\n", "                inputs=[index],\n", "                outputs = classifier.errors(y),\n", "                givens = {\n", "                        x: test_set_x[index*mini_batch_size:(index+1)*mini_batch_size],\n", "                        y: test_set_y[index*mini_batch_size:(index+1)*mini_batch_size]\n", "                        }\n", "        )\n", "        \n", "        validate_model = theano.function(\n", "            inputs=[index],\n", "            outputs=classifier.errors(y),\n", "            givens={\n", "                x: valid_set_x[index * mini_batch_size:(index + 1) * mini_batch_size],\n", "                y: valid_set_y[index * mini_batch_size:(index + 1) * mini_batch_size]\n", "            }\n", "        )\n", "        \n", "        train_model = theano.function(\n", "            inputs=[index],\n", "            outputs=cost,\n", "            updates=updates,\n", "            givens={\n", "                x: train_set_x[index * mini_batch_size: (index + 1) * mini_batch_size],\n", "                y: train_set_y[index * mini_batch_size: (index + 1) * mini_batch_size]\n", "            }\n", "        )\n", "            \n", "        \n", "        \n", "        for minibatch_index in range(n_train_batches):\n", "\n", "            #There are n_train_batches*mini_batch_size used for the training\n", "            minibatch_avg_cost = train_model(minibatch_index)\n", "            \n", "            print(\"Error function for minibatch %i/%i is %.5f\"%(minibatch_index+1, n_train_batches, minibatch_avg_cost))\n", "            # iteration number\n", "            iter = (epoch - 1) * n_train_batches + minibatch_index\n", "            \n", "            if (iter + 1) % validation_frequency == 0:\n", "                # compute zero-one loss on validation set\n", "                \n", "                validation_losses = [validate_model(i) for i\n", "                                     in range(n_valid_batches)]\n", "                \n", "                this_validation_loss = np.mean(validation_losses)\n", "\n", "                print(\n", "                    'epoch %i, minibatch %i/%i, validation error %f %%' %\n", "                    (\n", "                        epoch,\n", "                        minibatch_index + 1,\n", "                        n_train_batches,\n", "                        this_validation_loss * 100.\n", "                    )\n", "                )\n", "\n", "                # if we got the best validation score until now\n", "                if this_validation_loss < best_validation_loss:\n", "                    #improve patience if loss improvement is good enough\n", "                    if (\n", "                        this_validation_loss < best_validation_loss *\n", "                        improvement_threshold\n", "                    ):\n", "                        patience = max(patience, iter * patience_increase)\n", "\n", "                    best_validation_loss = this_validation_loss\n", "                    best_iter = iter\n", "\n", "                    # test it on the test set\n", "                    test_losses = [test_model(i) for i\n", "                                   in range(n_test_batches)]\n", "                    test_score = np.mean(test_losses)\n", "\n", "                    print(('     epoch %i, minibatch %i/%i, test error of '\n", "                           'best model %f %%') %\n", "                          (epoch, minibatch_index + 1, n_train_batches,\n", "                           test_score * 100.))\n", "\n", "            if patience <= iter:\n", "                done_looping = True\n", "                break\n", "\n", "    end_time = timeit.default_timer()\n", "    print(('Optimization complete. Best validation score of %f %% '\n", "           'obtained at iteration %i, with test performance %f %%') %\n", "          (best_validation_loss * 100., best_iter + 1, test_score * 100.))\n", "    print(('The code for file ' +\n", "           ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr)\n", "\n"], "cell_type": "code", "metadata": {"_uuid": "448c242b06e52f2a16f9178ffa9cb13c2a7b2c03", "collapsed": true, "_cell_guid": "2e6bf0af-8c54-4727-8cf7-d75a79752164"}, "execution_count": null}, {"outputs": [], "source": ["learning_rate = 0.01\n", "L1_reg = 0.00\n", "L2_reg = 0.0001\n", "n_epochs = 100\n", "batch_size = 10000\n", "mini_batch_size = 100\n", "n_kerns = (3,3)\n", "n_hidden = 750\n", "n_out = 5270\n", "\n", "test_cnn(learning_rate = learning_rate,\n", "         L1_reg = L1_reg,\n", "         L2_reg = L2_reg,\n", "         n_epochs = n_epochs,\n", "         batch_size = batch_size,\n", "         mini_batch_size = mini_batch_size,\n", "         n_kerns = n_kerns,\n", "         n_hidden = n_hidden,\n", "         n_out = n_out)"], "cell_type": "code", "metadata": {"_uuid": "f5e6ac69e6a9c0e67c6ec82b8e9a1791668eb32e", "collapsed": true, "_cell_guid": "443a006c-cf03-4537-9ed6-15143f0a85f2"}, "execution_count": null}, {"outputs": [], "source": [], "cell_type": "code", "metadata": {"_uuid": "7176047a03256bfc073fa586a18a1b27d779e286", "collapsed": true, "_cell_guid": "01d405db-0fde-4b62-a6e3-86350e49a3aa"}, "execution_count": null}], "nbformat": 4, "metadata": {"language_info": {"nbconvert_exporter": "python", "pygments_lexer": "ipython3", "name": "python", "file_extension": ".py", "mimetype": "text/x-python", "version": "3.6.3", "codemirror_mode": {"version": 3, "name": "ipython"}}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "nbformat_minor": 1}