{"cells": [{"source": ["import pymongo\n", "from pymongo import MongoClient\n", "client = MongoClient()\n", "client = MongoClient('localhost', 27017)\n", "# client = MongoClient('mongodb://localhost:27017/')\n", "db = client.db_cdiscount"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "06aa516b-4951-4209-a560-44c902f0b13b", "collapsed": true, "_uuid": "9f694ff835170739341ca36494c002f925690466"}}, {"source": ["import io\n", "import bson                       # this is installed with the pymongo package\n", "import matplotlib.pyplot as plt\n", "from skimage.data import imread   # or, whatever image library you prefer\n", "import multiprocessing as mp      # will come in handy due to the size of the data"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "7ed8a6ac-9162-4050-bb25-6fa407703294", "collapsed": true, "_uuid": "358a158d3beb3734e08777abdbba4357c3aed538"}}, {"source": ["import numpy as np\n", "import scipy\n", "import time"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "427f6b14-0c48-4563-b783-59eef84f1fc5", "collapsed": true, "_uuid": "0e15d5709865bbceb27dd322f648deab19952682"}}, {"source": ["from keras.applications.inception_v3 import InceptionV3\n", "from keras.preprocessing import image\n", "from keras.models import Model\n", "from keras.layers import Dense, GlobalAveragePooling2D\n", "from keras import backend as K"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "57db2ccb-7c90-4337-b6b7-365e5b90fc84", "collapsed": true, "_uuid": "2b0f97c622e520f6d9230f6fbfbb39e7850ac9c2"}}, {"source": ["# create the base pre-trained model\n", "base_model = InceptionV3(weights='imagenet', include_top=False)"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "da0c89c1-1a73-43fe-a846-f508d50fc56c", "collapsed": true, "_uuid": "5f4b18edae419b2923a2193b2948051cd54b18ee"}}, {"source": ["first = 0\n", "last = 10\n", "batch = 4\n", "num_batch = int(last/batch)\n", "# last = 82"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "14fa5cdd-0662-4e78-aeb0-ddcb5846a719", "collapsed": true, "_uuid": "1e46bb80157121e294148d5caf1165c9630d2415"}}, {"source": ["epochs = 10\n", "batch_size = 4\n", "num_classes = db.cat_encode.count()"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "1738491b-4a6c-4ffd-b407-38311041fd53", "collapsed": true, "_uuid": "49263d1743196e71a346ac4dd74ed1b15f59367a"}}, {"source": ["x = base_model.output\n", "x = GlobalAveragePooling2D()(x)\n", "# let's add a fully-connected layer\n", "x = Dense(1024, activation='relu')(x)\n", "# and a logistic layer -- let's say we have 9 class\n", "predictions = Dense(num_classes, activation='softmax')(x)"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "a58b116f-93f1-40e2-b36c-9b305e169324", "collapsed": true, "_uuid": "6e92468b1f5003437319d71c5cba23b511a9c9ba"}}, {"source": ["# this is the model we will train\n", "model = Model(inputs=base_model.input, outputs=predictions)"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "9d096388-077c-49bd-9773-ef87127249cf", "collapsed": true, "_uuid": "150975108862313b07554c1792632b51d0c49f0f"}}, {"source": ["# first: train only the top layers (which were randomly initialized)\n", "# i.e. freeze all convolutional InceptionV3 layers\n", "for layer in base_model.layers:\n", "    layer.trainable = False"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "c396103f-566a-4908-ae45-2703be3054c4", "collapsed": true, "_uuid": "20648503a0852b6eeece8bfb083915a8408d64ac"}}, {"source": ["# compile the model (should be done *after* setting layers to non-trainable)\n", "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "2d0b5f8d-4db3-4c0d-aad8-d55b547f8150", "collapsed": true, "_uuid": "4bc1ed4065b3f9b09024de1932cec46b75b8dc58"}}, {"source": ["start_time = time.time()\n", "print('start code')\n", "counter = 0\n", "row_count = 0\n", "for j in range(0,num_batch-1):\n", "# for j in range(0,1):\n", "    batch_time = time.time()\n", "    a1 = int(list(np.linspace(first,last,num_batch))[j])\n", "    an = int(list(np.linspace(first,last,num_batch))[j+1])\n", "    \n", "    lst_batch = []\n", "    \n", "    print(a1,an)\n", "    \n", "    cur = db.train.find({})[a1:an]\n", "    \n", "    m= an-a1\n", "        \n", "    i = 0\n", "#     pic_array\n", "    while (cur.alive):\n", "        idx = cur.next()\n", "        dic = {}\n", "        \n", "        category_id =idx['category_id'] \n", "    #     print(category_id)\n", "\n", "        cat = db.cat_encode.find_one({ \"cat\" : (category_id)}, {\"cat\" : 1.0, \"_id\" : 0})['cat']\n", "    #     print(cat)\n", "        \n", "        picture = (imread(io.BytesIO(idx['imgs'][0]['picture'])))\n", "        picture  = np.float32(scipy.misc.imresize(picture, (150,150), interp='bilinear', mode=None)/255.0)        \n", "        dic['picture'] = picture\n", "        \n", "        encode = db.cat_encode.find_one({ \"cat\" : (category_id)}, {\"encode\" : 1.0, \"_id\" : 0})['encode']\n", "    #     print(encode)\n", "        dic['encode'] = encode\n", "        \n", "        lst_batch.append(dic)\n", "        \n", "        i+=1\n", "    X_batch = np.array([lst['picture'] for lst in lst_batch])\n", "    Y_batch = np.array([lst['encode'] for lst in lst_batch])\n", "    print('X batch size: ', X_batch.shape)\n", "    print('Y batch size: ', Y_batch.shape)\n", "    \n", "    del(lst_batch)\n", "        \n", "    model.fit(X_batch, Y_batch, batch_size=batch_size, epochs=epochs, validation_split=0.3)\n", "    \n", "#     print(X_batch.shape[0])\n", "    row_count = row_count + X_batch.shape[0]\n", "    counter = counter + X_batch.shape[0]\n", "    \n", "    if(row_count >= 100000):\n", "        print('row count: ',row_count)\n", "        counter = counter + row_count\n", "        model_name = 'model_' + str(counter) +'.h5'\n", "        print(model_name)\n", "#         model.model.save('E://kaggle//Cdiscount//model//'+model_name)\n", "        print('Model saved')\n", "        row_count = 0\n", "    \n", "    print(\"--- %s seconds ---\" % (time.time() - batch_time))\n", "    print('rows completed: ', counter)\n", "    \n", "print(\"--- %s seconds ---\" % (time.time() - start_time))\n", "# model.model.save('E://kaggle//Cdiscount//model//final_model.h5')"], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "9967b1ed-e837-4a68-8a3c-56659cb1ba93", "collapsed": true, "_uuid": "26cfd04cf3d32c9b77fc4b9f4df52bfe0e655e80"}}, {"source": [], "outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "f800eec5-4114-4e47-9c55-6305ec2c12eb", "collapsed": true, "_uuid": "a88a2f7c9a39c791279d4f36443d161c171a9daf"}}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "anaconda-cloud": {}, "language_info": {"version": "3.6.3", "mimetype": "text/x-python", "file_extension": ".py", "pygments_lexer": "ipython3", "codemirror_mode": {"version": 3, "name": "ipython"}, "nbconvert_exporter": "python", "name": "python"}}}