{"nbformat": 4, "cells": [{"cell_type": "code", "outputs": [], "metadata": {"_uuid": "69d2f60ea815912daec71effa42a470c6074a258", "collapsed": true, "_cell_guid": "e4247924-275c-40c5-b1e2-d99c3628ff5c"}, "execution_count": null, "source": ["import pandas as pd\n", "import numpy as np\n", "import bson\n", "import io\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "from PIL import Image\n", "import tables ##enables hdf tables\n", "import cv2 #opencv helpful for storing image as array\n", "\n", "from matplotlib.colors import ListedColormap\n", "from wordcloud import WordCloud"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "f4819fe071e35294188487b0ca4581773b3943ab", "_cell_guid": "bc577e0e-1ebf-4895-a22c-9fb85d2c7460"}, "execution_count": null, "source": ["path = '../input/'\n", "!ls \"$path\""]}, {"cell_type": "markdown", "metadata": {"_uuid": "b098aa074fdefad0304399f1b48c454ecc0e6ef8", "_cell_guid": "fd9ab510-a956-484e-95af-a900c777ea45"}, "source": ["#### Read Files"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "903efc4617c0f6a533ca2301e790222702457e78", "collapsed": true, "_cell_guid": "ffd8a247-43e5-4b50-b331-bbc9fffe121f"}, "execution_count": null, "source": ["categories = pd.read_csv('{}{}'.format(path,'category_names.csv'),\n", "                         index_col=0)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "8b27132a34ceb37780f23c0f15179f0b557562e2", "_cell_guid": "111721b7-5ef2-4771-a847-3caa7c6d25a6"}, "execution_count": null, "source": ["categories.head()"]}, {"cell_type": "markdown", "metadata": {"_uuid": "56ea1ba90258d79ca7254fff2dde4a385321f6d4", "_cell_guid": "17fc803e-5350-4b20-bcbc-be348738d6c5"}, "source": ["#### Read bson file and convert to pandas DataFrame"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "4fa168dcd1047f38951ab529e2fb3d5bc633043b", "collapsed": true, "_cell_guid": "3dd561e5-409f-46d2-8159-fab560c5c36c"}, "execution_count": null, "source": ["with open('{}{}'.format(path,'train_example.bson'),'rb') as b:\n", "    df = pd.DataFrame(bson.decode_all(b.read()))"]}, {"cell_type": "markdown", "metadata": {"_uuid": "59ac341bdbf264c3bf5c378fdc19ab793e78046a", "_cell_guid": "02e5d02f-2680-4746-85ab-639d342824b7"}, "source": ["###### keep only binary image data in the imgs column"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "ad762506a3574b29d15517f07ccc5670e6252fe8", "collapsed": true, "_cell_guid": "f02e3778-e73b-41bd-a014-1d3b66b44bf3"}, "execution_count": null, "source": ["df['imgs'] = df['imgs'].apply(lambda rec: rec[0]['picture'])"]}, {"cell_type": "markdown", "metadata": {"_uuid": "081e9106652b3f91370c8d6ef315754f6a1872fd", "_cell_guid": "98c4856b-7044-4b15-b402-c40966fc0597"}, "source": ["#### set category_id as index"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "af77286fecb87ff6fb5256a1e2660a0cbf43479e", "collapsed": true, "_cell_guid": "5f895b30-612b-4895-aae8-b9a3cafa7592"}, "execution_count": null, "source": ["df.set_index('category_id',inplace=True)\n", "df.head()"]}, {"cell_type": "markdown", "metadata": {"_uuid": "47a5bcce645c212cdcca52839230c3bf116ed606", "_cell_guid": "89986407-285e-4acb-97e6-48097f9b16d4"}, "source": ["##### Combine images and categries"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "3b6516b515f1c8a02bb9df4c656608c52915fb93", "collapsed": true, "_cell_guid": "a7e2f197-26c2-4eb1-a168-6d0d9c844759"}, "execution_count": null, "source": ["df[categories.columns.tolist()] = categories.loc[df.index]"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "7a2285e3d78518a009dacb002caf1556fbad9e15", "collapsed": true, "_cell_guid": "092e0af0-112c-45bc-9a4d-722df7d8b2c7"}, "execution_count": null, "source": ["df.head()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "73997d2021a258d16b20d90e360c9cb428dda36f", "collapsed": true, "_cell_guid": "9458930c-82e7-4947-915d-2d2b9afcbd15"}, "execution_count": null, "source": ["df['imgs'] = df['imgs'].apply(lambda img: Image.open(io.BytesIO(img)))"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "187639202b8ba8f87c63d7899d707f961ee9cef7", "collapsed": true, "_cell_guid": "33293491-342e-49d3-bd01-0fd7e33e40a2"}, "execution_count": null, "source": ["fig,axs  = plt.subplots(7,5 ,figsize=(10,10))\n", "title = df['category_level1'].str.split('-').str[0].str.strip()\n", "# title += ','\n", "# title += df['category_level2'].str.split('-').str[0].str.strip()\n", "title = title.tolist()\n", "axs = axs.flatten()\n", "for i,ax in enumerate(axs):\n", "    ax.imshow(df.iloc[i,1],\n", "              interpolation='nearest', \n", "              aspect='auto')\n", "    ax.set_title(title[i])\n", "    #remove frame and ticks\n", "    ax.axis('off')\n", "plt.tight_layout()"]}, {"cell_type": "markdown", "metadata": {"_uuid": "f809a02bf0ea062d0c9841acda6ee2b35b07fafb", "collapsed": true, "_cell_guid": "0cf92dcb-c6e5-4a5d-8a6c-d142dfd78f1f"}, "source": ["### Overview of categories"]}, {"cell_type": "markdown", "metadata": {"_uuid": "f02ce75fc675d5737af4357f4a6b68a4264f6c92", "collapsed": true, "_cell_guid": "067e9514-1d7e-4cb5-afc6-2262c55874c2"}, "source": ["#### Read the train dataset"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "1bc6d65e540298531e3c00ecb2becc551db1ce33", "collapsed": true, "_cell_guid": "342339e6-4180-4b39-a905-72a1718807c9"}, "execution_count": null, "source": ["CHUNK_SIZE = 100000\n", "with open('{}{}'.format(path,'train.bson'),'rb',buffering=True) as b:\n", "    i=0\n", "    lst = []\n", "    for line in bson.decode_file_iter(b):\n", "        if i%CHUNK_SIZE == 0 and i !=0:\n", "            df = pd.DataFrame(lst)\n", "            # df['imgs'] = df['imgs'].apply(\n", "            #lambda reclst: ','.join(rec['picture'].hex() for rec in reclst))\n", "            try:\n", "                df.iloc[:,:-1].to_hdf('file.h5',\n", "                                      key='train',\n", "                                      format='table',\n", "                                      append=True)\n", "            except Exception as e:\n", "                #catch disk full\n", "                print('error: ',e)\n", "                break\n", "            lst=[]\n", "        lst.append(line)\n", "        i+=1"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "e97c91a87a20479827db70029e916732ee5ed114", "collapsed": true, "_cell_guid": "9c08a244-fc14-485c-8fff-a5db2789d558"}, "execution_count": null, "source": ["train = pd.read_hdf('file.h5',key='train')\n", "#combine with categries\n", "train = pd.merge(categories,train,right_on='category_id',left_index=True)"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "73306936d016b1cd92766475194a7e369e43bf96", "collapsed": true, "_cell_guid": "524cdc71-7155-49c5-8afb-2dd663449ec7"}, "execution_count": null, "source": ["train.info()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "69de352e4469e521e29802972e0f63e361933619", "collapsed": true, "_cell_guid": "fd1d2628-2a25-4653-b3c6-01e9622ba9b9"}, "execution_count": null, "source": ["train.head()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "a603170efbbe9b90d3aa7bb36119ee071c588987", "collapsed": true, "_cell_guid": "79bcdbc5-3789-468b-a188-3bb47d31df83"}, "execution_count": null, "source": ["train.shape"]}, {"cell_type": "markdown", "metadata": {"_uuid": "53745642b8d480b54fbfaceee0c9c7891d68a295", "_cell_guid": "ed05f960-bf5e-4e49-aa62-d7d7df4914d5"}, "source": ["#### see distinct categories"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "ef65a53540f560529b80cedd5b3f1b01defd3eaf", "collapsed": true, "_cell_guid": "f948d6c4-dd60-42a9-9923-8718abbadb96"}, "execution_count": null, "source": ["cats = train['category_level1'].value_counts()\n", "cats.head()"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "e8b8e13c7fda5eee361afcbb329fa4eaeaed34c1", "collapsed": true, "_cell_guid": "a1a3fe19-e00b-4ee2-92b7-e4a02e7b61d4"}, "execution_count": null, "source": ["abbriv = cats.index.str.split('\\W').str[0].str.strip()\n", "abbriv"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "a0bf43893e46b0c55c670fb9a48e4e6b8de302fc", "collapsed": true, "_cell_guid": "36695c4b-3a8d-489b-8975-915ce2e2f7ab"}, "execution_count": null, "source": ["sns.set_style('white')\n", "fig,ax = plt.subplots(1,figsize=(12,6))\n", "pal = ListedColormap(sns.color_palette('Paired').as_hex())\n", "colors = pal(np.interp(cats,[cats.min(),cats.max()],[0,1]))\n", "bars = ax.bar(range(1,len(cats)+1),cats,color=colors);\n", "ax.set_xticks([]);\n", "ax.set_xlim(0,len(cats))\n", "ax1 = plt.twiny(ax)\n", "ax1.set_xlim(0,len(cats))\n", "ax1.set_xticks(range(1,len(abbriv)+1,1));\n", "ax1.set_xticklabels(abbriv.values,rotation=90);\n", "# sns.despine();"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "a518b03951508eb30d971b8d9d5b6798972cfca6", "collapsed": true, "_cell_guid": "017ebf60-9c8d-4275-9c76-973091849eaf"}, "execution_count": null, "source": ["categories.head()"]}, {"cell_type": "raw", "metadata": {"_uuid": "9b03f2dc31dbf47efde4effcb5d590b93bddb7ce", "_cell_guid": "22a23dfc-4eec-4622-a8bc-e9e0357a1957"}, "source": []}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "ce60d847f49269651270ee36824f642012374c1a", "collapsed": true, "_cell_guid": "f54242ed-22d0-4107-be6c-cef0207b34f3"}, "execution_count": null, "source": ["wc = WordCloud(max_words=500,  margin=20,\n", "               random_state=0).generate(' '.join(categories.iloc[:,0].values.flatten()))"]}, {"cell_type": "code", "outputs": [], "metadata": {"_uuid": "d77a0d62d0a3fefaf92fc140d6c58205b2c7607f", "_cell_guid": "9d80175a-46ad-4942-be86-0eca9343370c"}, "execution_count": null, "source": ["def grey_color_func(word, font_size, position, orientation, random_state=None,\n", "                    **kwargs):\n", "    return \"hsl(0, 0%%, %d%%)\" % np.random.randint(60, 100)\n", "default_colors = wc.to_array()\n", "plt.title(\"Custom colors\")\n", "plt.imshow(wc.recolor(color_func=grey_color_func, random_state=3),\n", "           interpolation=\"bilinear\")\n", "wc.to_file(\"a_new_hope.png\")\n", "plt.axis(\"off\")\n", "plt.figure()\n", "plt.title(\"Default colors\")\n", "plt.imshow(default_colors, interpolation=\"bilinear\")\n", "plt.axis(\"off\")\n", "plt.show()"]}, {"cell_type": "code", "outputs": [], "metadata": {"collapsed": true}, "execution_count": null, "source": []}], "metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "version": "3.6.1", "name": "python", "nbconvert_exporter": "python", "mimetype": "text/x-python"}}, "nbformat_minor": 1}