{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "16daf9b5-f588-1152-8911-d0402ed3eb23"
      },
      "source": [
        "A simple 2 layer CNN to process MNIST dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "98da0e5b-9078-18dd-e6e9-84ce3a42d56b"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "mnist = pd.read_csv('../input/train.csv')\n",
        "mnist_test = pd.read_csv('../input/test.csv')\n",
        "\n",
        "x = tf.placeholder(tf.float32, [None, 784])\n",
        "y_ = tf.placeholder(tf.float32, [None, 10])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8b094f3b-dbed-af08-bb0b-bbe46be12d2f"
      },
      "source": [
        "### Creating the Weights and Biases"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "30d42c77-88cb-dd5a-b328-ab670338880c"
      },
      "source": [
        "#### For this I use truncated normal distributions for initialization of weights, to prevent 0 gradients. Also, since ReLU neurons will be used, the bias should be slightly positive to prevent 'dead neurons'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cf0fb7e1-6e85-67cc-b28b-d601e587357e"
      },
      "outputs": [],
      "source": [
        "def weight_variable(shape):\n",
        "    initial = tf.truncated_normal(shape, stddev = 0.1)\n",
        "    return tf.Variable(initial)\n",
        "\n",
        "def bias_variable(shape):\n",
        "    initial = tf.constant(0.1, shape=shape)\n",
        "    return tf.Variable(initial)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "970f6035-e3fe-c0a5-b7b5-381b1ec222c5"
      },
      "source": [
        "### Convolution and Pooling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "85ce0144-1e88-524e-0ba2-fa452a4250eb"
      },
      "source": [
        "#### Vanilla version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "73b56609-356c-951d-80aa-1051df0e0b96"
      },
      "outputs": [],
      "source": [
        "def conv2d(x, W):\n",
        "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
        "\n",
        "def max_pool(x):\n",
        "    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "784bbec3-1ced-e54e-c7f4-32c0a755d893"
      },
      "source": [
        "### First Convolutional Layer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "07d36448-4085-4c5a-a69b-d5e8f16bd01a"
      },
      "source": [
        "#### It will compute 32 feautures from every 5x5 patch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "68954e1c-1da1-61a7-86b9-2ce3d1f439a8"
      },
      "outputs": [],
      "source": [
        "#initailizing the weight and bias variables\n",
        "W_conv1 = weight_variable([5, 5, 1, 32])\n",
        "b_conv1 = bias_variable([32])\n",
        "#reshaping the input variable to a 4d tensor\n",
        "x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
        "#convolution and max pooling\n",
        "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
        "h_pool1 = max_pool(h_conv1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "f9d3521b-c4b9-be5d-1202-3f6d742ac54a"
      },
      "source": [
        "##### The image size is now 14x14 with 32 channels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "2cb27307-e290-f7f3-0435-bd35f88a95bb"
      },
      "source": [
        "### Second Convolutional Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d66b111b-872e-af65-97fb-a4090b145a53"
      },
      "outputs": [],
      "source": [
        "#initializing the weight and bias variables\n",
        "W_conv2 = weight_variable([5, 5, 32, 64])\n",
        "b_conv2 = bias_variable([64])\n",
        "\n",
        "#convolution and max pooling\n",
        "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
        "h_pool2 = max_pool(h_conv2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "c23caa14-5529-c58c-cd2d-62f89fc0635f"
      },
      "source": [
        "##### The image size is now 7x7 with 64 channels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "73567486-dad0-131a-c58f-8c7333989a47"
      },
      "source": [
        "### Densely Connected Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "044961fe-a42e-4d11-053a-aeb99add4c43"
      },
      "outputs": [],
      "source": [
        "W_fc1 = weight_variable([7*7*64, 1024])\n",
        "b_fc1 = bias_variable([1024])\n",
        "\n",
        "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
        "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "69dd9aed-0cb5-0262-a6e2-124547c228b5"
      },
      "source": [
        "### Dropout Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a963836f-e192-713e-b5be-bed6271ac892"
      },
      "outputs": [],
      "source": [
        "#prob that neuron's output should be kept during dropout\n",
        "keep_prob = tf.placeholder(tf.float32)\n",
        "h_fc1_dropout = tf.nn.dropout(h_fc1, keep_prob)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "033b71f2-6c01-2943-0baa-05176c388bd6"
      },
      "source": [
        "### Readout Layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d0739b10-0203-179a-73fb-4aa1e34e12fa"
      },
      "outputs": [],
      "source": [
        "W_fc2 = weight_variable([1024, 10])\n",
        "b_fc2 = bias_variable([10])\n",
        "\n",
        "y_conv = tf.matmul(h_fc1_dropout, W_fc2) + b_fc2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b63969e1-edb1-3f56-b417-529a4fae067a"
      },
      "outputs": [],
      "source": [
        "#Function to randomly select batches of training data\n",
        "def next_batch(num):\n",
        "    l = np.random.randint(0, 42000, num)\n",
        "    return (mnist.iloc[l, 1:], np.eye(10)[mnist.iloc[l, 0]])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "40666bc3-86a9-70b7-14e9-0c7df8b3dcbf"
      },
      "source": [
        "### Train and Evaluate the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "40efa304-4edc-4427-3f1b-73d131b89351"
      },
      "outputs": [],
      "source": [
        "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
        "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
        "predictions = tf.equal(tf.arg_max(y_, 1), tf.arg_max(y_conv, 1))\n",
        "accuracy = tf.reduce_mean(tf.cast(predictions, tf.float32))\n",
        "result = tf.arg_max(y_conv, 1)\n",
        "sess = tf.Session()\n",
        "sess.run(tf.global_variables_initializer())\n",
        "with sess.as_default():\n",
        "    for i in range(1000):\n",
        "        batch = next_batch(50)\n",
        "        if i%100 == 0:\n",
        "            train_accuracy = accuracy.eval(feed_dict={x:batch[0], y_:batch[1], keep_prob:0.5})\n",
        "            print (\"step %d has accuracy %g\"%(i, train_accuracy))\n",
        "        train_step.run(feed_dict={x:batch[0], y_:batch[1], keep_prob:0.5})\n",
        "    low = 0\n",
        "    arr = []\n",
        "    while (low <= (28000-50)):\n",
        "        arr.extend(result.eval(feed_dict={x:mnist_test.iloc[low:low+50, :], keep_prob:1.0}).tolist())\n",
        "        #print(\"Step: \\n\" + str(low))\n",
        "        low += 50\n",
        "    \n",
        "    y_test_labels = pd.DataFrame({\"ImageId\": list(range(1,len(arr)+1)),'Label': arr})\n",
        "    #y_test_labels.index += 1\n",
        "    #y_test_labels.columns.names = ['ImageId']\n",
        "    print (y_test_labels)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "73421d66-4fc1-39f6-f1f0-ce7362a1104d"
      },
      "outputs": [],
      "source": [
        "#from subprocess import check_output\n",
        "\n",
        "y_test_labels.to_csv('output.csv', index=False, header=True, sep = ',')\n",
        "#print(check_output([\"ls\", \"./\"]).decode(\"utf8\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "ef83ab67-ab4a-16ae-793a-7249430d6957"
      },
      "source": [
        "**My code does not generate a csv file yet. I'm still working on why this is not happening... I believe I have to use TensorFlow specific Variables and functions to make this work. Please provide suggestions/comments for this :)**"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}