{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"import os\nimport cv2\nimport glob\nimport math\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.utils import shuffle\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.cross_validation import train_test_split\n\nimport lasagne\nfrom lasagne.layers import helper\nfrom lasagne.updates import adam\nfrom lasagne.nonlinearities import rectify, softmax\nfrom lasagne.layers import InputLayer, MaxPool2DLayer, DenseLayer, DropoutLayer, helper\nfrom lasagne.layers import Conv2DLayer as ConvLayer\n\nimport theano\nfrom theano import tensor as T\n"},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"'''\nDoesn't run through kaggle scripts because of the 20 min limit. \nDownload the code, expand the network, add more iterations\n'''\n\n\n'''\nLoading data functions\n'''\nPIXELS = 24\nimageSize = PIXELS * PIXELS\nnum_features = imageSize \n\ndef load_train_cv(encoder):\n    X_train = []\n    y_train = []\n    print('Read train images')\n    for j in range(10):\n        print('Load folder c{}'.format(j))\n        path = os.path.join('..', 'input', 'train', 'c' + str(j), '*.jpg')\n        files = glob.glob(path)\n        for fl in files:\n            img = cv2.imread(fl,0)\n            img = cv2.resize(img, (PIXELS, PIXELS))\n            #img = img.transpose(2, 0, 1)\n            img = np.reshape(img, (1, num_features))\n            X_train.append(img)\n            y_train.append(j)\n\n    X_train = np.array(X_train)\n    y_train = np.array(y_train)\n\n    y_train = encoder.fit_transform(y_train).astype('int32')\n\n    X_train, y_train = shuffle(X_train, y_train)\n\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.1)\n\n    X_train = X_train.reshape(X_train.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_train, y_train, X_test, y_test, encoder\n\ndef load_test():\n    print('Read test images')\n    path = os.path.join('..', 'input', 'test', '*.jpg')\n    files = glob.glob(path)\n    X_test = []\n    X_test_id = []\n    total = 0\n    thr = math.floor(len(files)/10)\n    for fl in files:\n        flbase = os.path.basename(fl)\n        img = cv2.imread(fl,0)\n        img = cv2.resize(img, (PIXELS, PIXELS))\n        #img = img.transpose(2, 0, 1)\n        img = np.reshape(img, (1, num_features))\n        X_test.append(img)\n        X_test_id.append(flbase)\n        total += 1\n        if total%thr == 0:\n            print('Read {} images from {}'.format(total, len(files)))\n\n    X_test = np.array(X_test)\n    X_test_id = np.array(X_test_id)\n\n    X_test = X_test.reshape(X_test.shape[0], 1, PIXELS, PIXELS).astype('float32') / 255.\n\n    return X_test, X_test_id\n\n\n'''\nLasagne Model ZFTurboNet and Batch Iterator\n'''\ndef ZFTurboNet(input_var=None):\n    l_in = InputLayer(shape=(None, 1, PIXELS, PIXELS), input_var=input_var)\n\n    l_conv = ConvLayer(l_in, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_convb = ConvLayer(l_conv, num_filters=8, filter_size=3, pad=1, nonlinearity=rectify)\n    l_pool = MaxPool2DLayer(l_convb, pool_size=2) # feature maps 12x12\n\n    #l_dropout1 = DropoutLayer(l_pool, p=0.25)\n    l_hidden = DenseLayer(l_pool, num_units=128, nonlinearity=rectify)\n    #l_dropout2 = DropoutLayer(l_hidden, p=0.5)\n\n    l_out = DenseLayer(l_hidden, num_units=10, nonlinearity=softmax)\n\n    return l_out\n\ndef iterate_minibatches(inputs, targets, batchsize):\n    assert len(inputs) == len(targets)\n    indices = np.arange(len(inputs))\n    np.random.shuffle(indices)\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = indices[start_idx:start_idx + batchsize]\n        yield inputs[excerpt], targets[excerpt]\n"},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"BATCHSIZE = 32\nLR = 0.001\nITERS = 2\n\nX = T.tensor4('X')\nY = T.ivector('y')\n"},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"# set up theano functions to generate output by feeding data through network, any test outputs should be deterministic\noutput_layer = ZFTurboNet(X)\noutput_train = lasagne.layers.get_output(output_layer)\noutput_test = lasagne.layers.get_output(output_layer, deterministic=True)\n\n# set up the loss that we aim to minimize, when using cat cross entropy our Y should be ints not one-hot\nloss = lasagne.objectives.categorical_crossentropy(output_train, Y)\nloss = loss.mean()\n\n# set up loss functions for validation dataset\nvalid_loss = lasagne.objectives.categorical_crossentropy(output_test, Y)\nvalid_loss = valid_loss.mean()\n\nvalid_acc = T.mean(T.eq(T.argmax(output_test, axis=1), Y), dtype=theano.config.floatX)\n\n# get parameters from network and set up sgd with nesterov momentum to update parameters\nparams = lasagne.layers.get_all_params(output_layer, trainable=True)\nupdates = adam(loss, params, learning_rate=LR)\n\n# set up training and prediction functions\ntrain_fn = theano.function(inputs=[X,Y], outputs=loss, updates=updates)\nvalid_fn = theano.function(inputs=[X,Y], outputs=[valid_loss, valid_acc])\n\n# set up prediction function\npredict_proba = theano.function(inputs=[X], outputs=output_test)\n"},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"collapsed":false},"outputs":[],"source":"\n\"\"\"\nSet up all theano functions\n\"\"\"\n\n'''\nload training data and start training\n'''\n\n# load data\nX_test, X_test_id = load_test()\n\n# loop over training functions for however many iterations, print information while training\ntry:\n    for epoch in range(ITERS):\n        # do the training\n        start = time.time()\n        # training batches\n        train_loss = []\n        for batch in iterate_minibatches(train_X, train_y, BATCHSIZE):\n            inputs, targets = batch\n            train_loss.append(train_fn(inputs, targets))\n        train_loss = np.mean(train_loss)\n        # validation batches\n        valid_loss = []\n        valid_acc = []\n        for batch in iterate_minibatches(valid_X, valid_y, BATCHSIZE):\n            inputs, targets = batch\n            valid_eval = valid_fn(inputs, targets)\n            valid_loss.append(valid_eval[0])\n            valid_acc.append(valid_eval[1])\n        valid_loss = np.mean(valid_loss)\n        valid_acc = np.mean(valid_acc)\n        # get ratio of TL to VL\n        ratio = train_loss / valid_loss\n        end = time.time() - start\n        # print training details\n        print('iter:', epoch, '| TL:', np.round(train_loss,decimals=3), '| VL:', np.round(valid_loss, decimals=3), '| Vacc:', np.round(valid_acc, decimals=3), '| Ratio:', np.round(ratio, decimals=2), '| Time:', np.round(end, decimals=1))\n\nexcept KeyboardInterrupt:\n    pass\n\n'''\nMake Submission\n'''\n\n#make predictions\nprint('Making predictions')\nPRED_BATCH = 2\ndef iterate_pred_minibatches(inputs, batchsize):\n    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n        excerpt = slice(start_idx, start_idx + batchsize)\n        yield inputs[excerpt]\n\npredictions = []\nfor pred_batch in iterate_pred_minibatches(X_test, PRED_BATCH):\n    predictions.extend(predict_proba(pred_batch))\n\npredictions = np.array(predictions)\n\nprint('pred shape')\nprint(predictions.shape)\n\nprint('Creating Submission')\ndef create_submission(predictions, test_id):\n    result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])\n    result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)\n    result1.to_csv('submission_ZFTurboNet.csv', index=False)\n\ncreate_submission(predictions, X_test_id)\n"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":0}