{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "331e84b8-84ad-32cf-eae0-2a8a7221a804" }, "source": [ "This approach is inspired by MNIST tutorials. \n", "\n", "This model packs some good techniques like \n", "\n", " 1. CNN, to retain the shape information instead of spreading the data into an\u200b array of 784 pixels\n", " 2. Drop out, to fix the overfitting of data\n", " 3. Batch Normalisation, to normalise the data for better accuracy and higher learning rates." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bba4d28e-56f0-abc2-005b-6fa6f628b98d" }, "outputs": [], "source": [ "# libraries and variables\n", "\n", "import math\n", "import numpy as np\n", "import pandas as pd\n", "\n", "#To display plots in cell rather than in seperate window\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "import matplotlib.cm as cm\n", "\n", "import tensorflow as tf" ] }, { "cell_type": "heading", "metadata": { "_cell_guid": "8b665a3e-bc22-ade7-84ac-c97af603fcc5" }, "source": [ "Data Extraction, Visualization and labelling" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f4ca3d51-6c70-2f07-4eb1-e042770c283b" }, "outputs": [], "source": [ "# Reading the data\n", "data = pd.read_csv(\"../input/train.csv\")\n", "print(data.shape)\n", "print(data.head())" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f496ef03-ac35-ec9d-5265-e4cdc0e72e6e" }, "source": [ "Here each picture is streched into single array containing the pixel values" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ad94e7e5-12c6-a900-7f59-fa4f56b2137c" }, "outputs": [], "source": [ "images = data.iloc[:,1:].values #Selecting subset, i.e, from column 1\n", "images = images.astype(np.float) #Converting the values to float as in next step we divide by 255\n", "\n", "#converting from (0,255) to (0,1)\n", "images = np.multiply(images, 1.0/255.0)\n", "print('images({0[0]},{0[1]})'.format(images.shape))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b822da0a-e06d-37d4-ebbf-6bea1fefedc6" }, "source": [ "Let's display the image" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "81c9ce90-51d2-6c4e-60f3-ce8290f5341c" }, "outputs": [], "source": [ "def display_image(row):\n", " image = row.reshape(28,28)\n", " plt.axis('off')\n", " plt.imshow(image, cmap=cm.binary) #To show the image in binary colors, black and white\n", "\n", "display_image(images[6])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f03aa3ef-2000-799f-2904-4e4a1d03c742" }, "source": [ "Let's retrieve the labels" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "490051eb-bf97-7214-d48d-89717230a4f5" }, "outputs": [], "source": [ "lables = data[[0]].values.ravel() #used to return a view, in contrast to returning a copy in flatten\n", "print('Numbers of labels: {0}'.format(len(lables)))\n", "print('label for [{0}] is: {1}'.format(8,lables[8]))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a0c54eeb-1617-b214-f650-4686499cba8d" }, "source": [ "Lets convert labels to vectors, as they will be useful going forward for loss function" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "95f00a05-1d22-0f98-eb83-325c47e8ccfc" }, "outputs": [], "source": [ "def flat_to_oneHot(lables_param,unique_label_Number):\n", " print(lables_param.shape[0])\n", " number_of_labels = lables_param.shape[0]\n", " #creating 10 spaces, for storing and marking the actual digit as 1\n", " index_offset = np.arange(number_of_labels) * unique_label_Number\n", " labels_one_hot = np.zeros((number_of_labels,unique_label_Number))\n", " labels_one_hot.flat[index_offset+lables_param.ravel()] = 1\n", " return labels_one_hot\n", "\n", "labels_count = 10\n", "lables_oneHot_format = flat_to_oneHot(lables,labels_count)\n", "\n", "print('labels({0[0]},{0[1]})'.format(lables_oneHot_format.shape))\n", "print('label[{0}] : {1}'.format(17,lables_oneHot_format[17]))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f9de0d67-24e9-8638-62ea-dd50edcd6231" }, "source": [ "Now let's seperate data for training and validation" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dc4f5e48-9584-0629-d4e5-ee0a64d07692" }, "outputs": [], "source": [ "train_images = images\n", "train_labels = lables_oneHot_format\n", "print(train_labels)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c15ddae2-82ec-249f-92a4-1156ba5347ae" }, "source": [ "Now that data is ready let's jump to Tensorflow programming of NN" ] }, { "cell_type": "heading", "metadata": { "_cell_guid": "2ba77ac1-3ef2-ab87-2af1-7e468ac2b0eb" }, "source": [ "Tensor Flow graph" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a686ae57-3ef6-f386-1594-8daf090f938a" }, "outputs": [], "source": [ "#Initialization\n", "tf.set_random_seed(4)\n", "X = tf.placeholder(tf.float32, [None, 784])\n", "\n", "#For the correct answers\n", "Y_ = tf.placeholder(tf.float32, [None, 10])\n", "\n", "#For learning rate\n", "lr = tf.placeholder(tf.float32)\n", "\n", "# test flag for batch normalization\n", "tst = tf.placeholder(tf.bool)\n", "iter = tf.placeholder(tf.int32)\n", "\n", "# dropout probability\n", "pkeep = tf.placeholder(tf.float32)\n", "pkeep_conv = tf.placeholder(tf.float32)\n", "\n", "def batchnorm(Ylogits, is_test, iteration, offset, convolutional=False):\n", " exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, iteration) # adding the iteration prevents from averaging across non-existing iterations\n", " bnepsilon = 1e-5\n", " if convolutional:\n", " mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])\n", " else:\n", " mean, variance = tf.nn.moments(Ylogits, [0])\n", " update_moving_everages = exp_moving_avg.apply([mean, variance])\n", " m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean)\n", " v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance)\n", " Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)\n", " return Ybn, update_moving_everages\n", "\n", "def no_batchnorm(Ylogits, is_test, iteration, offset, convolutional=False):\n", " return Ylogits, tf.no_op()\n", "\n", "def compatible_convolutional_noise_shape(Y):\n", " noiseshape = tf.shape(Y)\n", " noiseshape = noiseshape * tf.constant([1,0,0,1]) + tf.constant([0,1,1,0])\n", " return noiseshape\n", "\n", "def max_pool(x):\n", " return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c74ae077-5ad4-ba35-1902-52de4d7084ee" }, "outputs": [], "source": [ "#here this NN consists of 3 layers\n", "A = 32\n", "B = 64\n", "C = 1024\n", "\n", "#Batch size\n", "BATCH_SIZE = 100\n", "\n", "#Initializing with small random values\n", "W1 = tf.Variable(tf.truncated_normal([6, 6, 1, A], stddev=0.1))\n", "B1 = tf.Variable(tf.constant(0.1, tf.float32, [A]))\n", "W2 = tf.Variable(tf.truncated_normal([5,5,A, B], stddev=0.1))\n", "B2 = tf.Variable(tf.constant(0.1, tf.float32, [B]))\n", "\n", "W3 = tf.Variable(tf.truncated_normal([7*7*B, C], stddev=0.1))\n", "B3 = tf.Variable(tf.constant(0.1, tf.float32, [C]))\n", "W4 = tf.Variable(tf.truncated_normal([C, 10], stddev=0.1))\n", "B4 = tf.Variable(tf.constant(0.1, tf.float32, [10]))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "5deea968-fc76-e935-e9f6-700fba12a2b9" }, "source": [ "To overcome overfitting, dropout is used. So for all the convolutional layers there will be dropouts after the activation function in each layer. Let's build the model with the drop out layers. For the model it is always benificial to use Relu Activation function instead of sigmoid funtion." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0ca65f4e-5aae-2781-bccc-fa85f5d255d9" }, "source": [ "Now that we have weights and biases for all the layers. lets formulate the model for all layers.\n", "In Each layer logits will undergo convolution first and then batchnorm. This result is sent to activation function. The result from activation function is sent to drop out function, to drop out few neurons for the next layer inputs." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "582b048d-0c0f-f2bf-0c05-5565fa44430b" }, "outputs": [], "source": [ "# The model\n", "\n", "image1 = tf.reshape(X, [-1,28 , 28,1])\n", "Y1C = tf.nn.conv2d(image1, W1, strides=[1, 1, 1, 1], padding='SAME') + B1\n", "Y1bn, update_ema1 = batchnorm(Y1C, tst, iter, B1, convolutional=True)\n", "Y1R = tf.nn.relu(Y1bn)\n", "Y1 = tf.nn.dropout(Y1R, pkeep_conv, compatible_convolutional_noise_shape(Y1R))\n", "P1 = max_pool(Y1)\n", "\n", "Y2C = tf.nn.conv2d(P1, W2, strides=[1, 1, 1, 1], padding='SAME') + B2\n", "Y2bn, update_ema2 = batchnorm(Y2C, tst, iter, B2, convolutional=True)\n", "Y2R = tf.nn.relu(Y2bn)\n", "Y2 = tf.nn.dropout(Y2R, pkeep_conv, compatible_convolutional_noise_shape(Y2R))\n", "P2 = max_pool(Y2)\n", "\n", "\n", "# reshape the output from the third convolution for the fully connected layer\n", "YY = tf.reshape(P2, shape=[-1, 7 * 7 * B])\n", "Y3C = tf.matmul(YY, W3) + B3\n", "Y4bn, update_ema3 = batchnorm(Y3C, tst, iter, B3)\n", "Y3R = tf.nn.relu(Y4bn)\n", "Y3 = tf.nn.dropout(Y3R, pkeep)\n", "\n", "\n", "Ylogits = tf.matmul(Y3, W4) + B4\n", "Y = tf.nn.softmax(Ylogits)\n", "\n", "update_ema = tf.group(update_ema1, update_ema2, update_ema3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df2da61d-d90e-08e8-a1c2-ca7ab0b15428" }, "outputs": [], "source": [ "#For calculating loss function\n", "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)\n", "cross_entropy = tf.reduce_mean(cross_entropy)*100\n", "\n", "#Calculating the accuaracy of trained model\n", "train_predicted_output = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))#checks for equal indexes\n", "accuracy = tf.reduce_mean(tf.cast(train_predicted_output, tf.float32))\n", "#print(accuracy)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2cf91a62-16a8-ab80-55d7-4461212f63f6" }, "outputs": [], "source": [ "#Code for creating a new batch\n", "\n", "epochs_completed = 0\n", "index_in_epoch = 0\n", "num_examples = train_images.shape[0]\n", "\n", "# serve data by batches\n", "def next_batch(batch_size):\n", "\n", " global train_images\n", " global train_labels\n", " global index_in_epoch\n", " global epochs_completed\n", "\n", " start = index_in_epoch\n", " index_in_epoch += batch_size\n", "\n", " # when all trainig data have been already used, it is reorder randomly\n", " if index_in_epoch > num_examples:\n", " # finished epoch\n", " epochs_completed += 1\n", " # shuffle the data\n", " perm = np.arange(num_examples)\n", " np.random.shuffle(perm)\n", " train_images = train_images[perm]\n", " train_labels = train_labels[perm]\n", " # start next epoch\n", " start = 0\n", " index_in_epoch = batch_size\n", " assert batch_size <= num_examples\n", " end = index_in_epoch\n", " #print(train_images[start:end])\n", " return train_images[start:end], train_labels[start:end]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a594e9b5-3668-505d-100b-ea905d17105d" }, "outputs": [], "source": [ "# training step, the learning rate is a placeholder\n", "train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)\n", "\n", "# starting the tensor flow session\n", "init = tf.global_variables_initializer()\n", "sess = tf.Session()\n", "sess.run(init)\n", "\n", "for i in range(1500):\n", "\n", " #print(i)\n", " # training on batches of 100 images with 100 labels\n", " batch_X, batch_Y = next_batch(BATCH_SIZE)\n", "\n", "\n", " # learning rate decay\n", " max_learning_rate = 0.02\n", " min_learning_rate = 0.0001\n", " decay_speed = 1600\n", " learning_rate = min_learning_rate + (max_learning_rate - min_learning_rate) * math.exp(-i/decay_speed)\n", " # the backpropagation training step\n", " sess.run(train_step, {X: batch_X, Y_: batch_Y, lr: learning_rate, tst: False, pkeep: 0.75, pkeep_conv: 1.0})\n", " sess.run(update_ema, {X: batch_X, Y_: batch_Y, tst: False, iter: i, pkeep: 1.0, pkeep_conv: 1.0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d048c140-63ee-ebbb-dc92-87afbccf36e1" }, "outputs": [], "source": [ "#After the model is trained, lets test it on a test data set\n", "test_images = pd.read_csv(\"../input/test.csv\")\n", "test_images = test_images.iloc[:,:].values\n", "test_images = test_images.astype(np.float)\n", "\n", "# convert from [0:255] => [0.0:1.0]\n", "test_images = np.multiply(test_images, 1.0 / 255.0)\n", "\n", "print('test_images({0[0]},{0[1]})'.format(test_images.shape))\n", "print(test_images.shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "39f9d26d-eb7e-a2ae-d091-33b4e1dd3d33" }, "outputs": [], "source": [ "# predict test set\n", "#predicted_lables = predict.eval(feed_dict={x: test_images, keep_prob: 1.0})\n", "predict = tf.argmax(Y,1)\n", "# using batches is more resource efficient\n", "predicted_lables = np.zeros(test_images.shape[0])\n", "for i in range(0, test_images.shape[0] // BATCH_SIZE):\n", " predicted_lables[i * BATCH_SIZE: (i + 1) * BATCH_SIZE] = predict.eval(session=sess,\n", " feed_dict={X: test_images[i * BATCH_SIZE: (i + 1) * BATCH_SIZE],\n", " pkeep: 1.0, tst: False, pkeep_conv: 1.0})\n", "\n", "#print('predicted_lables({0})'.format(len(predicted_lables)))\n", "\n", "display_image(test_images[275])\n", "print('predicted_lables[{0}] => {1}'.format(11,predicted_lables[11]))\n", "\n", "# saving the results\n", "np.savetxt('Digit_Recogniser.csv',\n", " np.c_[range(1,len(test_images)+1),predicted_lables],\n", " delimiter=',',\n", " header = 'ImageId,Label',\n", " comments = '',\n", " fmt='%d')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f8f21286-ca2b-f528-d7b8-dcaa4f7eb7df" }, "outputs": [], "source": [ "sess.close()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8aa2eecf-de4a-71c6-3771-8498fd514117" }, "outputs": [], "source": [ "This model achieved an accuracy of 0.99143\n", "Also note that, some part of the code has been adapted from Martin Gorner tutorial for deep learning and tensor flow" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }