{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "'''\nDoesn't run through kaggle scripts because of the 20 min limit. \nDownload the code, expand the network, add more iterations\n'''\n\nimport os\nimport cv2\nimport glob\nimport math\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.cross_validation import train_test_split\n\nimport lasagne\nfrom lasagne.layers import helper\nfrom lasagne.updates import adam\nfrom lasagne.nonlinearities import rectify, softmax\nfrom lasagne.layers import InputLayer, MaxPool2DLayer, DenseLayer, DropoutLayer, helper\nfrom lasagne.layers import Conv2DLayer as ConvLayer\n\nimport theano\nfrom theano import tensor as T\n\n'''\nLoading data functions\n'''\nPIXELS = 16\nimageSize = PIXELS * PIXELS\nnum_features = imageSize \n\ndef load_train_cv(encoder):\n    X_train = []\n    y_train = []\n    print('Read train images')\n    for j in range(10):\n        print('Load folder c{}'.format(j))\n        path = os.path.join('..', 'input', 'train', 'c' + str(j), '*.jpg')\n        files = glob.glob(path)\n        for fl in files:\n            img = cv2.imread(fl,0)\n            img = cv2.resize(img, (PIXELS, PIXELS))\n            #img = img.transpose(2, 0, 1)\n            img = np.reshape(img, (1, num_features))\n            X_train.append(img)\n            y_train.append(j)\n\n    X_train = np.array(X_train)\n    y_train = np.array(y_train)\n\n    y_train = encoder.fit_transform(y_train).astype('int32')\n\n    X_train, y_train = shuffle(X_train, y_train)\n\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1)\n\n    X_train = X_train.reshape(X_train.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_train, y_train, X_test, y_test, encoder\n\ndef load_test():\n    print('Read test images')\n    path = os.path.join('..', 'input', 'test', '*.jpg')\n    files = glob.glob(path)\n    X_test = []\n    X_test_id = []\n    total = 0\n    thr = math.floor(len(files)/10)\n    for fl in files:\n        flbase = os.path.basename(fl)\n        img = cv2.imread(fl,0)\n        img = cv2.resize(img, (PIXELS, PIXELS))\n        #img = img.transpose(2, 0, 1)\n        img = np.reshape(img, (1, num_features))\n        X_test.append(img)\n        X_test_id.append(flbase)\n        total += 1\n        if total%thr == 0:\n            print('Read {} images from {}'.format(total, len(files)))\n\n    X_test = np.array(X_test)\n    X_test_id = np.array(X_test_id)\n\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_test, X_test_id\n\n\n'''\nLasagne Model ZFTurboNet and Batch Iterator\n'''\ndef ZFTurboNet(input_var=None):\n    l_in = InputLayer(shape=(None, 1, PIXELS, PIXELS), input_var=input_var)\n\n    l_conv = ConvLayer(l_in, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_convb = ConvLayer(l_conv, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_pool = MaxPool2DLayer(l_convb, pool_size=2) # feature maps 12x12\n\n    #l_dropout1 = DropoutLayer(l_pool, p=0.25)\n    l_hidden = DenseLayer(l_pool, num_units=128, nonlinearity=rectify)\n    #l_dropout2 = DropoutLayer(l_hidden, p=0.5)\n\n    l_out = DenseLayer(l_hidden, num_units=10, nonlinearity=softmax)\n\n    return l_out\n\ndef iterate_minibatches(inputs, targets, batchsize):\n    assert len(inputs) == len(targets)\n    indices = np.arange(len(inputs))\n    np.random.shuffle(indices)\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = indices[start_idx:start_idx + batchsize]\n        yield inputs[excerpt], targets[excerpt]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "\"\"\"\nSet up all theano functions\n\"\"\"\nBATCHSIZE = 32\nLR = 0.001\nITERS = 2\n\nX = T.tensor4('X')\nY = T.ivector('y')\n\n# set up theano functions to generate output by feeding data through network, any test outputs should be deterministic\noutput_layer = ZFTurboNet(X)\noutput_train = lasagne.layers.get_output(output_layer)\noutput_test = lasagne.layers.get_output(output_layer, deterministic=True)\n\n# set up the loss that we aim to minimize, when using cat cross entropy our Y should be ints not one-hot\nloss = lasagne.objectives.categorical_crossentropy(output_train, Y)\nloss = loss.mean()\n\n# set up loss functions for validation dataset\nvalid_loss = lasagne.objectives.categorical_crossentropy(output_test, Y)\nvalid_loss = valid_loss.mean()\n\nvalid_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)\n\n# get parameters from network and set up sgd with nesterov momentum to update parameters\nparams = lasagne.layers.get_all_params(output_layer, trainable=True)\nupdates = adam(loss, params, learning_rate=LR)\n\n# set up training and prediction functions\ntrain_fn = theano.function(inputs=[X,Y], outputs=loss, updates=updates)\nvalid_fn = theano.function(inputs=[X,Y], outputs=[valid_loss, valid_acc])\n\n# set up prediction function\npredict_proba = theano.function(inputs=[X], outputs=output_test)\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "'''\nload training data and start training\n'''\nencoder = LabelEncoder()\n\n# load the training and validation data sets\ntrain_X, train_y, valid_X, valid_y, encoder = load_train_cv(encoder)\nprint('Train shape:', train_X.shape, 'Test shape:', valid_X.shape)\n\n# load data\nX_test, X_test_id = load_test()\n\n"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Training"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# loop over training functions for however many iterations, print information while training\ntry:\n    for epoch in range(ITERS):\n        # do the training\n        start = time.time()\n        # training batches\n        train_loss = []\n        for batch in iterate_minibatches(train_X, train_y, BATCHSIZE):\n            inputs, targets = batch\n            train_loss.append(train_fn(inputs, targets))\n        train_loss = np.mean(train_loss)\n        # validation batches\n        valid_loss = []\n        valid_acc = []\n        for batch in iterate_minibatches(valid_X, valid_y, BATCHSIZE):\n            inputs, targets = batch\n            valid_eval = valid_fn(inputs, targets)\n            valid_loss.append(valid_eval[0])\n            valid_acc.append(valid_eval[1])\n        valid_loss = np.mean(valid_loss)\n        valid_acc = np.mean(valid_acc)\n        # get ratio of TL to VL\n        ratio = train_loss / valid_loss\n        end = time.time() - start\n        # print training details\n        print('iter:', epoch, '| TL:', np.round(train_loss,decimals=3), '| VL:', np.round(valid_loss, decimals=3), '| Vacc:', np.round(valid_acc, decimals=3), '| Ratio:', np.round(ratio, decimals=2), '| Time:', np.round(end, decimals=1))\n\nexcept KeyboardInterrupt:\n    pass\n\n"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Make Submission"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "\n\n#make predictions\nprint('Making predictions')\nPRED_BATCH = 2\ndef iterate_pred_minibatches(inputs, batchsize):\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = slice(start_idx, start_idx + batchsize)\n        yield inputs[excerpt]\n\npredictions = []\nfor pred_batch in iterate_pred_minibatches(X_test, PRED_BATCH):\n    predictions.extend(predict_proba(pred_batch))\n\npredictions = np.array(predictions)\n\nprint('pred shape')\nprint(predictions.shape)\n\nprint('Creating Submission')\ndef create_submission(predictions, test_id):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    result1.to_csv('submission_ZFTurboNet.csv', index=False)\n\ncreate_submission(predictions, X_test_id)"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}