{"cells": [{"source": ["### ran on google cloud datalab\n", "### won't work on kaggle kernel b/c it takes too long. "], "cell_type": "markdown", "metadata": {"_cell_guid": "17bb8b67-4ee5-4ba0-843b-c3d61633dd12", "_uuid": "817ab0f65f8f5ac7330c46dbd29e9c4847f08d84"}}, {"source": ["> ## random access code inspired by [this kernel](http://www.kaggle.com/am1to2/random-images-access-through-bson)"], "cell_type": "markdown", "metadata": {"_cell_guid": "20574208-1df8-4ccd-a57e-717176ee0234", "_uuid": "1f3354d8ce8279d1b1dc09b372fb257fe42f4fe7"}}, {"outputs": [], "source": ["from __future__ import print_function\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "import io\n", "import bson\n", "import matplotlib.pyplot as plt\n", "from skimage.data import imread   # or, whatever image library you prefer\n", "from sklearn.cross_validation import train_test_split\n", "import random\n", "from sklearn.externals import joblib\n", "from sklearn import neural_network"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "f9d636e7-aca5-479b-9e2a-c89e4da5443c", "_uuid": "74d5e4e3e41487f7f74880a85e6bf559013db1f6"}}, {"outputs": [], "source": ["def extract_categories_df(num_images):\n", "    img_category = list()\n", "    item_locs_list = list()\n", "    items_len_list = list()\n", "    pic_ind_list = list()\n", "    prod_id_list = list()\n", "\n", "    with open('../cdisount/train.bson', 'rb') as f:\n", "        data = bson.decode_file_iter(f)\n", "        last_item_loc = 0\n", "        item_len = 0\n", "        for c, d in enumerate(data):\n", "            loc = f.tell()\n", "            item_len = loc - last_item_loc\n", "            category_id = d['category_id']\n", "            prod_id = d[\"_id\"]\n", "\n", "            for e, pic in enumerate(d['imgs']):\n", "                prod_id_list.append(prod_id)\n", "                img_category.append(category_id)\n", "                item_locs_list.append(last_item_loc)\n", "                items_len_list.append(item_len)\n", "                pic_ind_list.append(e)\n", "                \n", "                if num_images is not None:\n", "                    if len(img_category) >= num_images:\n", "                        break\n", "            \n", "            last_item_loc = loc\n", "            \n", "            if num_images is not None:\n", "                if len(img_category) >= num_images:\n", "                    break\n", "    \n", "    f.close()\n", "    df_dict = {\n", "        'category': img_category,\n", "        \"prod_id\": prod_id_list,\n", "        \"img_id\": range(len(img_category)),\n", "        \"item_loc\": item_locs_list,\n", "        \"item_len\": items_len_list,\n", "        \"pic_ind\": pic_ind_list\n", "    }\n", "    df = pd.DataFrame(df_dict)\n", "    df.to_csv(\"all_images_categories.csv\", index=False)\n", "        \n", "    return df\n", "\n", "def get_image(image_id,data_df,fh):\n", "    img_info = data_df[data_df[\"img_id\"] == image_id]\n", "    item_loc = img_info[\"item_loc\"].values[0]\n", "    item_len = img_info[\"item_len\"].values[0]\n", "    pic_ind = img_info[\"pic_ind\"].values[0]\n", "    fh.seek(item_loc)\n", "    item_data = fh.read(item_len)\n", "    d = bson.BSON.decode(item_data)\n", "    \n", "    picture = imread(io.BytesIO(d[\"imgs\"][pic_ind]['picture']))\n", "    return picture"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "50387356-fe9e-4cd2-83ca-48cd3b5de95a", "_uuid": "55fac06e034620b84d6a581af6361011ae8eb419"}}, {"outputs": [], "source": ["cat_df = extract_categories_df(None)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "31bfb056-57ed-4bb9-9b33-c983bea6f268", "_uuid": "99823e7011f6ab1c9c00a8f6f49337877c8ce89e"}}, {"outputs": [], "source": ["train_fh = open('../cdisount/train.bson', 'rb')"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "4e839cb9-f286-4354-b03e-66a35906c599", "_uuid": "814b5988af31f6836420a423ee269f0a530a175a"}}, {"outputs": [], "source": ["X_images = np.zeros((10000,180,180,3)) #m images are 180 by 180 by 3\n", "X_ids = []\n", "Y_prod_ids = []"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "9592e17b-28d8-4ff8-964a-7bc049507f8a", "_uuid": "ac773d0eef3d5b33eec08dad28abbf86b3286b51"}}, {"outputs": [], "source": ["# extract 10000 images for training.\n", "num = 0\n", "for i in random.sample(range(len(cat_df)),10000):\n", "    pic = get_image(i,cat_df,train_fh)\n", "    X_images[num] = pic\n", "    X_ids.append(cat_df['prod_id'][i])\n", "    Y_prod_ids.append(cat_df['category'][i])\n", "    num = num + 1\n", "    \n", "    #show update every 1,000 images\n", "    if num > 0 and num % 1000 == 0:\n", "        print(\"[INFO] processed {}/{}\".format(num, 10000))"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "fe4260df-3aaf-45d3-8d9c-4fdda4a0a927", "_uuid": "ec0d7ad2c0309d917a33e775f40e8dc681a38a40"}}, {"outputs": [], "source": ["# get number of unique categories\n", "Y_df = pd.DataFrame({'product_id':Y_prod_ids})\n", "len(Y_df[\"product_id\"].unique())"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "d691df02-6da5-4809-8b4a-89dd58dedb5c", "_uuid": "260082868cbcfa19d733881ecd2cbc029e1bf72e"}}, {"outputs": [], "source": ["# flatten images\n", "X_images = X_images.reshape(X_images.shape[0], -1)\n", "X_images = X_images/255"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "21674d11-9c1f-43c5-b8a2-d02230b23031", "_uuid": "7a841c3623c7d54a26306d0d04aae96a405323cf"}}, {"outputs": [], "source": ["# partition the data into training and testing splits, using 80%\n", "# of the data for training and the remaining 20% for testing\n", "(X_train, X_test, Y_train, Y_test) = train_test_split(X_images, Y_prod_ids, test_size=0.20, random_state=42)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "1fdcb844-139b-431c-b7ce-a778a89f0414", "_uuid": "ad30eb2eb06fc0cb78e489ef573b72efc354e6be"}}, {"source": ["# Test model"], "cell_type": "markdown", "metadata": {"_cell_guid": "d3e9a004-6533-4073-84f2-c11c4eb788ac", "_uuid": "89caaaf4ddc9073e680d08306355e5562ac8ec7d"}}, {"outputs": [], "source": ["# Now, your classifier is 'Artificial Neural Network of 10 layers'\n", "# solver: one of lbfgs(quasi-Newton method), sgd (stochastic gradient descent), adam (a special stochasitc gradient decent); default is adam\n", "# alpha: L2 penalty(regularization term) parameter\n", "# hidden_layer_sizes: a tuple; the ith element represents the number of neurons in the ith hidden layer\n", "# activation: one of identity, logistic, tanh, relu; relu is default\n", "clf = neural_network.MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(10,), random_state=1).fit(X_train, Y_train)\n", "print(\"Artificial Neural Network(ANN)-10\")\n", "print(clf.score(X_test, Y_test))"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "a3506aaa-2fb3-4b80-8986-90978be3547e", "_uuid": "044da19494360a8480ef1545da61d8a155b0149d"}}, {"outputs": [], "source": ["# save the model to disk\n", "joblib.dump(clf, '/ANN_model.sav')\n", " \n", "# some time later...\n", " \n", "# load the model from disk\n", "# loaded_model = joblib.load(filename)\n", "# result = loaded_model.score(X_test, Y_test)\n", "# print(result)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "9dbc06ed-3c15-44ea-be12-a75a4485d3ce", "_uuid": "dee17350485c26bf68a3af37bfebbae5113a958d"}}, {"source": ["# Keep training model\n", "If train data is too big, results in memory error.\n", "So I decided to save the model, and then make a loop that deletes exisitng batch, generates random batches, and calls / trains the saved model.\n", "Below is how I generated 10 batches to train my model."], "cell_type": "markdown", "metadata": {"_cell_guid": "04c8cdc7-5407-47e5-a914-daa0aec7fe2b", "_uuid": "f3fcecb30338a23bdc0a529d13d5cbc0f9644726"}}, {"outputs": [], "source": ["trial = 1\n", "\n", "while trial < 11:\n", "  \n", "  print('Starting trial', trial)\n", "  X_images = []\n", "  X_images = np.zeros((10000,180,180,3)) #m images are 180 by 180 by 3\n", "  X_ids = []\n", "  Y_prod_ids = []\n", "\n", "  num = 0\n", "  for i in random.sample(range(len(cat_df)),10000):\n", "    pic = get_image(i,cat_df,train_fh)\n", "    X_images[num] = pic\n", "    X_ids.append(cat_df['prod_id'][i])\n", "    Y_prod_ids.append(cat_df['category'][i])\n", "    num = num + 1\n", "    \n", "    #show update every 2,500 images\n", "    if num > 0 and num % 2500 == 0:\n", "      print(\"[INFO] processed {}/{}\".format(num, 10000))\n", "\n", "  # flatten images\n", "  X_images = X_images.reshape(X_images.shape[0], -1)\n", "  X_images = X_images/255\n", "  \n", "  # partition the data into training and testing splits, using 90%\n", "  # of the data for training and the remaining 10% for testing\n", "  (X_train, X_test, Y_train, Y_test) = train_test_split(X_images, Y_prod_ids, test_size=0.10)\n", " \n", "\n", "  loaded_model = joblib.load('../cdiscount/ANN_model.sav')\n", "  clf = loaded_model.fit(X_train, Y_train)\n", "  print(\"Artificial Neural Network(ANN)-10\")\n", "  print(clf.score(X_test, Y_test))\n", "  joblib.dump(clf, '../cdiscount/ANN_model.sav')\n", "  \n", "  print('trial', trial, 'completed')\n", "  trial = trial + 1"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "077a1f6e-ed08-4f53-9a50-2cf54816ecbe", "_uuid": "4002c7e0ce56e87db5b8c31a9085689d648501dc"}}, {"source": ["# Make predictions"], "cell_type": "markdown", "metadata": {"_cell_guid": "d05f8a0b-f9aa-47ac-ba53-d0c1979348b2", "_uuid": "495cc7e592c4bccc29071884a9f32a705cab1c8c"}}, {"outputs": [], "source": ["def extract_test_df(num_images):\n", "    prod_id_list = list()\n", "    item_locs_list = list()\n", "    items_len_list = list()\n", "    pic_ind_list = list()\n", "\n", "    with open('../cdiscount/test.bson', 'rb') as f:\n", "        data = bson.decode_file_iter(f)\n", "        last_item_loc = 0\n", "        item_len = 0\n", "        for c, d in enumerate(data):\n", "            loc = f.tell()\n", "            item_len = loc - last_item_loc\n", "            prod_id = d[\"_id\"]\n", "\n", "            for e, pic in enumerate(d['imgs']):\n", "                prod_id_list.append(prod_id)\n", "                item_locs_list.append(last_item_loc)\n", "                items_len_list.append(item_len)\n", "                pic_ind_list.append(e)\n", "                \n", "                if num_images is not None:\n", "                    if len(prod_id) >= num_images:\n", "                        break\n", "            \n", "            last_item_loc = loc\n", "            \n", "            if num_images is not None:\n", "                if len(prod_id) >= num_images:\n", "                    break\n", "    \n", "    f.close()\n", "    df_dict = {\n", "        'prod_id': prod_id_list,\n", "        \"img_id\": range(len(prod_id_list)),\n", "        \"item_loc\": item_locs_list,\n", "        \"item_len\": items_len_list,\n", "        \"pic_ind\": pic_ind_list\n", "    }\n", "    df = pd.DataFrame(df_dict)\n", "    df.to_csv(\"../cdiscount/all_test_images_categories.csv\", index=False)\n", "        \n", "    return df"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "1cd7aaae-400a-4c21-b1d3-7a1f05e87822", "_uuid": "e57c3369fc5becebd8e666913e50c042a169b394"}}, {"outputs": [], "source": ["test_cat_df = extract_test_df(None) # pd.read_csv(\"../cdiscount/all_test_images_categories.csv\")"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "1c27835d-83e7-4acd-9be6-a4aa36298b67", "_uuid": "d8c3416efa77a12ff3543bc5d90c759a6161cba0"}}, {"outputs": [], "source": ["test_fh = open('../cdiscount/test.bson', 'rb')"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "7d8182cb-f7b6-4aa4-9baf-996f548e1f04", "_uuid": "232f415615e938fd925825026d46b1fa470566c1"}}, {"outputs": [], "source": ["test_cat_df.head()"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "b294d75d-91e0-4362-ac97-a3a3c8b0f6c6", "_uuid": "e92b4b193faef28234e2dac133f0cf9cc114aeba"}}, {"outputs": [], "source": ["len(test_cat_df)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "4e5c0002-cee4-49df-9fee-4350a7939a08", "_uuid": "4b8e02b021ac7a8d08a09ca024d334b474f0921f"}}, {"outputs": [], "source": ["test_cat_df.drop_duplicates(subset='prod_id', inplace=True)\n", "len(test_cat_df)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "36be247e-e4c3-4cd0-8d32-8f052e479723", "_uuid": "4c107dbcf5be945420cb32ed48dd73e556634f6e"}}, {"outputs": [], "source": ["test_cat_df.head()"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "b0cd1da0-b387-49c6-bdfb-c5624408702f", "_uuid": "1a188efb2990e9bf95a64546e9d41f6c48365954"}}, {"outputs": [], "source": ["test_cat_df = test_cat_df.drop('img_id', 1)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "d4e45ca1-0683-4558-83be-dec67a5e21bc", "_uuid": "6d542bc9c2b2589a701e5a2890f94af17684faec"}}, {"outputs": [], "source": ["test_cat_df.head()"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "1ac77ffe-a8c0-4cc1-bb4d-818c1eb59cef", "_uuid": "84fa1f23ca492a90155c9195562fc313060d0c66"}}, {"outputs": [], "source": ["new_entry = list(range(1768182)) \n", "test_cat_df = test_cat_df.assign(img_id = new_entry)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "8f1579c8-f3c8-48ea-86ad-6d0c4188243d", "_uuid": "28d58822d9bdce7a1e544987e7b798f78dd8c435"}}, {"outputs": [], "source": ["test_cat_df = test_cat_df[['img_id', 'item_len', 'item_loc', 'pic_ind', 'prod_id']]"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "80e7c094-a89a-470b-acac-39543cf57b2c", "_uuid": "90ae56672877ab4a70c728d420f7a9e5633f77d4"}}, {"outputs": [], "source": ["test_cat_df.head()"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "83ad2213-664b-44fc-9258-583b4c7ee80d", "_uuid": "1cd61e94a5b9a6adf54a6b30a9f1991e35ff2694"}}, {"outputs": [], "source": ["test_cat_df.to_csv(\"../cdiscount/all_test_images_categories.csv\", index=False)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "1f07f505-b102-4652-af5b-229ea3260750", "_uuid": "1f16079dbf99069a9de0fd7c636cf4b4771f3a21"}}, {"outputs": [], "source": ["mul = 20000\n", "phase = 1\n", "\n", "category_id = []\n", "loaded_model = joblib.load('../cdiscount/ANN_model.sav')\n", "\n", "while mul < 1768182:\n", "\n", "  print('Starting prediction', phase, '/ 89')\n", "  X_images = []\n", "  clf = []\n", "  X_images = np.zeros((20000,180,180,3)) #m images are 180 by 180 by 3\n", "  \n", "  i = 0\n", "  num = 0\n", "  for i in range(mul-20000, mul):\n", "    pic = get_image(i,test_cat_df,test_fh)\n", "    X_images[num] = pic\n", "    num = num + 1\n", "    \n", "  # flatten images\n", "  X_images = X_images.reshape(X_images.shape[0], -1)\n", "  X_images = X_images/255\n", "  clf = loaded_model.predict(X_images)\n", "  \n", "  numb = 0\n", "  while numb < len(clf):\n", "    category_id.append(clf[numb])\n", "    numb = numb + 1\n", "  \n", "  phase = phase + 1\n", "  mul = mul + 20000\n", "\n", "  \n", "print('Starting prediction', phase, '/ 89')\n", "X_images = []\n", "clf = []\n", "X_images = np.zeros((8182,180,180,3)) #m images are 180 by 180 by 3\n", "\n", "i = 0\n", "num = 0\n", "for i in range(mul-20000, 1768182):\n", "  pic = get_image(i,test_cat_df,test_fh)\n", "  X_images[num] = pic\n", "  num = num + 1\n", "  \n", "# flatten images\n", "X_images = X_images.reshape(X_images.shape[0], -1)\n", "X_images = X_images/255\n", "  \n", "clf = loaded_model.predict(X_images)\n", "  \n", "numb = 0\n", "while numb < len(clf):\n", "  category_id.append(clf[numb])\n", "  numb = numb + 1"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "c448c448-ffbe-48ad-934e-8dc8ad8fdad8", "_uuid": "05bec45567e7b641fd3baeb504a51adac1c68792"}}, {"outputs": [], "source": ["_id = test_cat_df['prod_id']"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "7f41be69-7bcb-4f11-9a7b-ff202b31e472", "_uuid": "0a4dfc517e22d51afaf939f01a41a133bff38c94"}}, {"outputs": [], "source": ["submission_df = test_cat_df.assign(category_id = category_id)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "578c688d-bed5-4928-b49c-ec09168a88fc", "_uuid": "81594724c3071220963414cba0d55643cafd3f3c"}}, {"outputs": [], "source": ["submission_df = submission_df.drop('img_id', 1)\n", "submission_df = submission_df.drop('item_len', 1)\n", "submission_df = submission_df.drop('item_loc', 1)\n", "submission_df = submission_df.drop('pic_ind', 1)\n", "submission_df.columns = ['_id', 'category_id']\n", "submission_df.head()"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true}}, {"outputs": [], "source": ["submission_df.to_csv(\"../cdiscount/my_submission.csv.gz\", compression=\"gzip\", index=False)"], "execution_count": null, "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "0df5becf-e332-41f8-a2c2-ae1c06158844", "_uuid": "9fa1add96c46bf644c95f6026d62511d438ca550"}}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "language_info": {"file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "version": "3.6.3", "pygments_lexer": "ipython3"}}}