{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import os\nimport cv2\nimport glob\nimport math\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.cross_validation import train_test_split\n\nimport lasagne\nfrom lasagne.layers import helper\nfrom lasagne.updates import adam\nfrom lasagne.nonlinearities import rectify, softmax\nfrom lasagne.layers import InputLayer, MaxPool2DLayer, DenseLayer, DropoutLayer, helper\nfrom lasagne.layers import Conv2DLayer as ConvLayer\n\nimport theano\nfrom theano import tensor as T\n\n'''\nLoading data functions\n'''\nPIXELS = 24\nimageSize = PIXELS * PIXELS\nnum_features = imageSize \n\ndef load_train_cv(encoder):\n    X_train = []\n    y_train = []\n    print('Read train images')\n    for j in range(10):\n        print('Load folder c{}'.format(j))\n        path = os.path.join('..', 'input', 'train', 'c' + str(j), '*.jpg')\n        files = glob.glob(path)\n        for fl in files:\n            img = cv2.imread(fl,0)\n            img = cv2.resize(img, (PIXELS, PIXELS))\n            #img = img.transpose(2, 0, 1)\n            img = np.reshape(img, (1, num_features))\n            X_train.append(img)\n            y_train.append(j)\n\n    X_train = np.array(X_train)\n    y_train = np.array(y_train)\n\n    y_train = encoder.fit_transform(y_train).astype('int32')\n\n    X_train, y_train = shuffle(X_train, y_train)\n\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1)\n\n    X_train = X_train.reshape(X_train.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_train, y_train, X_test, y_test, encoder\n\ndef load_test():\n    print('Read test images')\n    path = os.path.join('..', 'input', 'test', '*.jpg')\n    files = glob.glob(path)\n    X_test = []\n    X_test_id = []\n    total = 0\n    thr = math.floor(len(files)/10)\n    for fl in files:\n        flbase = os.path.basename(fl)\n        img = cv2.imread(fl,0)\n        img = cv2.resize(img, (PIXELS, PIXELS))\n        #img = img.transpose(2, 0, 1)\n        img = np.reshape(img, (1, num_features))\n        X_test.append(img)\n        X_test_id.append(flbase)\n        total += 1\n        if total%thr == 0:\n            print('Read {} images from {}'.format(total, len(files)))\n\n    X_test = np.array(X_test)\n    X_test_id = np.array(X_test_id)\n\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_test, X_test_id\n\n\n'''\nLasagne Model ZFTurboNet and Batch Iterator\n'''\ndef ZFTurboNet(input_var=None):\n    l_in = InputLayer(shape=(None, 1, PIXELS, PIXELS), input_var=input_var)\n\n    l_conv = ConvLayer(l_in, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_convb = ConvLayer(l_conv, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_pool = MaxPool2DLayer(l_convb, pool_size=2) # feature maps 12x12\n\n    #l_dropout1 = DropoutLayer(l_pool, p=0.25)\n    l_hidden = DenseLayer(l_pool, num_units=128, nonlinearity=rectify)\n    #l_dropout2 = DropoutLayer(l_hidden, p=0.5)\n\n    l_out = DenseLayer(l_hidden, num_units=10, nonlinearity=softmax)\n\n    return l_out\n\ndef iterate_minibatches(inputs, targets, batchsize):\n    assert len(inputs) == len(targets)\n    indices = np.arange(len(inputs))\n    np.random.shuffle(indices)\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = indices[start_idx:start_idx + batchsize]\n        yield inputs[excerpt], targets[excerpt]\n\n\"\"\"\nSet up all theano functions\n\"\"\"\nBATCHSIZE = 32\nLR = 0.001\nITERS = 2\n\nX = T.tensor4('X')\nY = T.ivector('y')\n\n# set up theano functions to generate output by feeding data through network, any test outputs should be deterministic\noutput_layer = ZFTurboNet(X)\noutput_train = lasagne.layers.get_output(output_layer)\noutput_test = lasagne.layers.get_output(output_layer, deterministic=True)\n\n# set up the loss that we aim to minimize, when using cat cross entropy our Y should be ints not one-hot\nloss = lasagne.objectives.categorical_crossentropy(output_train, Y)\nloss = loss.mean()\n\n# set up loss functions for validation dataset\nvalid_loss = lasagne.objectives.categorical_crossentropy(output_test, Y)\nvalid_loss = valid_loss.mean()\n\nvalid_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)\n\n# get parameters from network and set up sgd with nesterov momentum to update parameters\nparams = lasagne.layers.get_all_params(output_layer, trainable=True)\nupdates = adam(loss, params, learning_rate=LR)\n\n# set up training and prediction functions\ntrain_fn = theano.function(inputs=[X,Y], outputs=loss, updates=updates)\nvalid_fn = theano.function(inputs=[X,Y], outputs=[valid_loss, valid_acc])\n\n# set up prediction function\npredict_proba = theano.function(inputs=[X], outputs=output_test)\n\n'''\nload training data and start training\n'''\nencoder = LabelEncoder()\n\n# load the training and validation data sets\ntrain_X, train_y, valid_X, valid_y, encoder = load_train_cv(encoder)\nprint('Train shape:', train_X.shape, 'Test shape:', valid_X.shape)\n\n# load data\nX_test, X_test_id = load_test()\n\n# loop over training functions for however many iterations, print information while training\ntry:\n    for epoch in range(ITERS):\n        # do the training\n        start = time.time()\n        # training batches\n        train_loss = []\n        for batch in iterate_minibatches(train_X, train_y, BATCHSIZE):\n            inputs, targets = batch\n            train_loss.append(train_fn(inputs, targets))\n        train_loss = np.mean(train_loss)\n        # validation batches\n        valid_loss = []\n        valid_acc = []\n        for batch in iterate_minibatches(valid_X, valid_y, BATCHSIZE):\n            inputs, targets = batch\n            valid_eval = valid_fn(inputs, targets)\n            valid_loss.append(valid_eval[0])\n            valid_acc.append(valid_eval[1])\n        valid_loss = np.mean(valid_loss)\n        valid_acc = np.mean(valid_acc)\n        # get ratio of TL to VL\n        ratio = train_loss / valid_loss\n        end = time.time() - start\n        # print training details\n        print('iter:', epoch, '| TL:', np.round(train_loss,decimals=3), '| VL:', np.round(valid_loss, decimals=3), '| Vacc:', np.round(valid_acc, decimals=3), '| Ratio:', np.round(ratio, decimals=2), '| Time:', np.round(end, decimals=1))\n\nexcept KeyboardInterrupt:\n    pass\n\n'''\nMake Submission\n'''\n\n#make predictions\nprint('Making predictions')\nPRED_BATCH = 2\ndef iterate_pred_minibatches(inputs, batchsize):\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = slice(start_idx, start_idx + batchsize)\n        yield inputs[excerpt]\n\npredictions = []\nfor pred_batch in iterate_pred_minibatches(X_test, PRED_BATCH):\n    predictions.extend(predict_proba(pred_batch))\n\npredictions = np.array(predictions)\n\nprint('pred shape')\nprint(predictions.shape)\n\nprint('Creating Submission')\ndef create_submission(predictions, test_id):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    result1.to_csv('submission_ZFTurboNet.csv', index=False)\n\ncreate_submission(predictions, X_test_id)"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}