{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport math\n\nimport tensorflow as tf\nfrom tensorflow.python.framework import ops\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d24f213ef9cb5b95f2289e2ebf20c563b1d5e1a"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n\nX_train = train.iloc[:40000, 1:].values\nY_train = train.iloc[:40000, 0].values\nX_val = train.iloc[40000:, 1:].values\nY_val = train.iloc[40000:, 0].values\n\nX_test = test.iloc[:].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5421d6466233630ae1159a4b62ef123f45492140"},"cell_type":"code","source":"# Reshape X to (28,28)\n\nX_train_res = np.reshape(X_train, (X_train.shape[0], 28, 28, 1))\nX_val_res = np.reshape(X_val, (X_val.shape[0], 28, 28, 1))\nX_test_res = np.reshape(X_test, (X_test.shape[0], 28, 28, 1))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Create one_hot matrix to Y\n\ndef convert_one_hot(Y):\n    \n    Y_one = tf.one_hot(Y, depth=10, axis=-1)\n    \n    return Y_one","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10dedaaada21212d75493bdba1b1a73bbb75f6d2"},"cell_type":"code","source":"Y_train_1h = convert_one_hot(Y_train)\nY_val_1h = convert_one_hot(Y_val)\n\nwith tf.Session() as sess:\n    Y_train_one = sess.run(Y_train_1h)\n    Y_val_one = sess.run(Y_val_1h)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6240ab551f13dc2675757d6f3f730293de51821d"},"cell_type":"code","source":"print(\"X_train_res shape \" + str(X_train_res.shape))\nprint(\"X_val_res shape \" + str(X_val_res.shape))\nprint(\"X_test_res shape \" + str(X_test_res.shape))\nprint(\"Y_train_one shape \" + str(Y_train_one.shape))\nprint(\"Y_val_one shape \" + str(Y_val_one.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fee0a0e6bfb101d4480bd9c2fd66f1329ed148f0"},"cell_type":"code","source":"# Create placeholder for X, Y\n\ndef create_placeholders(n_W, n_H, n_C, n_Y):\n    # n_W: Width of the image\n    # n_H: Height of the image\n    # n_C: Layer of the image\n    # n_Y: Number of class\n    \n    X = tf.placeholder(dtype=tf.float32, shape=[None, n_W, n_H, n_C])\n    Y = tf.placeholder(dtype=tf.float32, shape=[None, n_Y])\n    \n    return X, Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3426f2c5da260ec5bbcaf479fb7f6e706d7fd703"},"cell_type":"code","source":"# Initialize weight parameter\n\ndef initialize_parameters():\n    \n    W1 = tf.get_variable(\"W1\", [8, 8, 1, 8], dtype = tf.float32, initializer = tf.contrib.layers.xavier_initializer())\n    W2 = tf.get_variable(\"W2\", [2, 2, 8, 16], dtype = tf.float32, initializer = tf.contrib.layers.xavier_initializer())\n\n    parameters = {\"W1\": W1,\n                  \"W2\": W2}\n    \n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd53f594bf1676183140518bcfacc03601c6f68d"},"cell_type":"code","source":"# Forward propagation\n\ndef forward_propagation(X, parameters):\n\n    #Implements the forward propagation for the model:\n    #CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED\n    \n    W1 = parameters['W1']\n    W2 = parameters['W2']\n    \n    # CONV2D: stride of 1, padding 'SAME'\n    Z1 = tf.nn.conv2d(X, W1, strides = [1, 1, 1, 1], padding = 'SAME')\n    # RELU\n    A1 = tf.nn.relu(Z1)\n    # MAXPOOL: window 8x8, sride 8, padding 'SAME'\n    P1 = tf.nn.max_pool(A1, ksize = [1, 8, 8, 1], strides = [1, 8, 8, 1], padding = 'SAME')\n    # CONV2D: filters W2, stride 1, padding 'SAME'\n    Z2 = tf.nn.conv2d(P1, W2, strides = [1, 1, 1, 1], padding = 'SAME')\n    # RELU\n    A2 = tf.nn.relu(Z2)\n    # MAXPOOL: window 4x4, stride 4, padding 'SAME'\n    P2 = tf.nn.max_pool(A2, ksize = [1, 4, 4, 1], strides = [1, 4, 4, 1], padding = 'SAME')\n    # FLATTEN\n    P2 = tf.contrib.layers.flatten(P2)\n    # FULLY-CONNECTED without non-linear activation function (not not call softmax).\n    # 6 neurons in output layer. Hint: one of the arguments should be \"activation_fn=None\" \n    Z3 = tf.contrib.layers.fully_connected(P2, 10, activation_fn = None)\n\n    return Z3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58b85265aaa4d45727df2f4d7ca6d36aa43587e5"},"cell_type":"code","source":"def compute_cost(Z3, Y):\n\n    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = Z3, labels = Y))\n    \n    return cost","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af49ad00e00d635b36522a0d22cc8030cda44dea"},"cell_type":"code","source":"def model(X_train, Y_train, X_val, Y_val, learning_rate = 0.009, num_epochs = 101, minibatch_size = 128, print_cost = True):\n    \"\"\"\n    Implements a three-layer ConvNet in Tensorflow:\n    CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED\n    \n    Arguments:\n    X_train -- training set, of shape (None, 64, 64, 1)\n    Y_train -- test set, of shape (None, n_y = 10)\n    X_test -- training set, of shape (None, 64, 64, 1)\n    Y_test -- test set, of shape (None, n_y = 10)\n    learning_rate -- learning rate of the optimization\n    num_epochs -- number of epochs of the optimization loop\n    minibatch_size -- size of a minibatch\n    print_cost -- True to print the cost every 100 epochs\n    \n    Returns:\n    train_accuracy -- real number, accuracy on the train set (X_train)\n    test_accuracy -- real number, testing accuracy on the test set (X_test)\n    parameters -- parameters learnt by the model. They can then be used to predict.\n    \"\"\"  \n    \n    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables \n    seed = 0\n    (m, n_H, n_W, n_C) = X_train.shape             \n    n_y = Y_train.shape[1]                            \n    costs = []                                        # To keep track of the cost\n    \n    # Create Placeholders of the correct shape\n    X, Y = create_placeholders(n_H, n_W, n_C, n_y)\n\n    # Initialize parameters\n    parameters = initialize_parameters()\n    \n    # Forward propagation\n    Z3 = forward_propagation(X, parameters)\n    \n    # Cost function\n    cost = compute_cost(Z3, Y)\n    \n    # Backpropagation\n    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)\n    \n    init = tf.global_variables_initializer()\n    \n    with tf.Session() as sess:\n        sess.run(init)\n        \n        for epoch in range(num_epochs):\n\n            minibatch_cost = 0.\n            num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set\n            seed += 1\n\n            for minibatch in range(num_minibatches):\n\n                minibatch_X = X_train[minibatch*minibatch_size:(minibatch+1)*minibatch_size, :, :, :]\n                minibatch_Y = Y_train[minibatch*minibatch_size:(minibatch+1)*minibatch_size, :]                \n                _ , temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})                \n                minibatch_cost += temp_cost / num_minibatches\n                \n\n            # Print the cost every epoch\n            if print_cost == True and epoch % 5 == 0:\n                print (\"Cost after epoch %i: %f\" % (epoch, minibatch_cost))\n            if print_cost == True and epoch % 1 == 0:\n                costs.append(minibatch_cost)\n        \n        \n        # plot the cost\n        plt.plot(np.squeeze(costs))\n        plt.ylabel('cost')\n        plt.xlabel('iterations (per tens)')\n        plt.title(\"Learning rate =\" + str(learning_rate))\n        plt.show()\n\n        # Calculate the correct predictions\n        predict_op = tf.argmax(Z3, 1)\n        correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))\n        \n        # Calculate accuracy on the test set\n        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n        print(accuracy)\n        train_accuracy = accuracy.eval({X: X_train, Y: Y_train})\n        test_accuracy = accuracy.eval({X: X_val, Y: Y_val})\n        print(\"Train Accuracy:\", train_accuracy)\n        print(\"Test Accuracy:\", test_accuracy)\n                \n        return train_accuracy, test_accuracy, parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1fede3c1fae4f36a314c821bee5d615156fce7e","scrolled":false},"cell_type":"code","source":"_, _, parameters = model(X_train_res, Y_train_one, X_val_res, Y_val_one)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}