{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"x_train=pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a76f8fdc249f6d24601b5e5bc44cb16edeaf8dd"},"cell_type":"code","source":"x_test_data=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b373a7c419860f837b10fdae4dad44cc77728949"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94d1b88a3f45c954b2b1c594124e9b6bb4a397dc"},"cell_type":"code","source":"x1=x_train.iloc[:,1:785]\ny1=x_train.iloc[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"079f46c8518b1e0b1d47238c26776e197b29f74c"},"cell_type":"code","source":"x_train,x_test,y_train,y_test=train_test_split(x1,y1,test_size=0.10,random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdb148e85e7f66d43491ad0edeaf5605de3ef6d8"},"cell_type":"code","source":"x_train.shape,x_test.shape,y_train.shape,y_test.shape,x_test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b34acb228aabbf202ae3b68801184acb6daf4da0"},"cell_type":"code","source":"x_train=x_train/255\nx_test=x_test/255\nx_test_data=x_test_data/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0df2094784afb129df08c3d67e1b87d27fb365"},"cell_type":"code","source":"x_train.shape,x_test.shape,x_test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c0701fe02fd44ae0b3f0954a86af168a16a5754"},"cell_type":"code","source":"import tensorflow as tf\nfrom keras.utils import to_categorical ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"707c14f661b607c22cc469117ae5b12801061bc8"},"cell_type":"code","source":"classes=10\ny_train=to_categorical(y_train,num_classes = classes)\ny_test=to_categorical(y_test,num_classes = classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d34e4371cebac1e6a74a2cc9ef14826dbb96ec19"},"cell_type":"code","source":"y_train[0] #x_train=tf.to_float(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c79504268cb8c5b3ed1cbf4b18c04100cde5537c"},"cell_type":"code","source":"epochs=30\nbatch_size=64\ndisplay_progress=40\nwt_init=tf.contrib.layers.xavier_initializer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90124a1970432f294acf961a835b96bc4f43d3dd"},"cell_type":"code","source":"n_input=784\n\nn_conv_1=32\nk_conv_1=3\n\nn_conv_2=64\nk_conv_2=3\n\nn_conv_3=128\nk_conv_3=3\n\npool_size=2\nmp_dropout=0.25\n\nn_dense=128\ndense_dropout=0.5\n\nn_dense2=128\n\nn_classes=10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"682ae753807c75fc41a6aef2459abb2c2aa4e4e5"},"cell_type":"code","source":"x=tf.placeholder(tf.float32,[None,n_input])\ny=tf.placeholder(tf.float32,[None,n_classes])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7a965dbc2404a5e980577692c20c5e605186628"},"cell_type":"code","source":"def dense(x,W,b):\n    z=tf.add(tf.matmul(x,W),b)\n    a=tf.nn.relu(z)\n    return a\n\ndef conv_2d(x,W,b,stride_length=1):\n    xw=tf.nn.conv2d(x,W,strides=[1,stride_length,stride_length,1],padding='SAME')\n    z=tf.nn.bias_add(xw,b)\n    a=tf.nn.relu(z)\n    return a\n\ndef maxpooling2d(x,p_size):\n    return tf.nn.max_pool(x,\n                         ksize=[1,p_size,p_size,1],\n                         strides=[1,p_size,p_size,1],\n                         padding='SAME')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9fafc69317d223d8e42ea6783d1165d04537ec9"},"cell_type":"code","source":"def network(x,weights,biases,n_in,mp_size,mp_dropout,den_dropout):\n    \n    sqr_dim=int(np.sqrt(n_in))\n    sqr_x=tf.reshape(x,shape=[-1,sqr_dim,sqr_dim,1])\n    \n    conv1=conv_2d(sqr_x,weights['W_c1'],biases['b_c1'])\n    conv2=conv_2d(conv1,weights['W_c2'],biases['b_c2'])\n    conv3=conv_2d(conv2,weights['W_c3'],biases['b_c3'])\n    conv4=conv_2d(conv3,weights['W_c4'],biases['b_c4'])\n    pool1=maxpooling2d(conv4,mp_size)\n    pool1=tf.nn.dropout(pool1,1-mp_dropout)\n    \n    conv5=conv_2d(pool1,weights['W_c5'],biases['b_c5'])\n    conv6=conv_2d(conv5,weights['W_c6'],biases['b_c6'])\n    conv7=conv_2d(conv6,weights['W_c7'],biases['b_c7'])\n    conv8=conv_2d(conv7,weights['W_c8'],biases['b_c8'])\n    pool2=maxpooling2d(conv8,mp_size)\n    pool2=tf.nn.dropout(pool2,1-mp_dropout)\n    #conv9=conv_2d(pool2,weights['W_c5'],biases['b_c5'])\n    \n    #conv9=conv_2d(pool2,weights['W_c9'],biases['b_c9'])\n    #conv10=conv_2d(conv9,weights['W_c10'],biases['b_c10'])\n    #conv11=conv_2d(conv10,weights['W_c11'],biases['b_c11'])\n    #conv12=conv_2d(conv11,weights['W_c12'],biases['b_c12'])\n    #pool3=maxpooling2d(conv12,mp_size)\n    #pool3=tf.nn.dropout(pool3,1-mp_dropout)\n    \n    flat = tf.reshape(pool2, [-1, weights['W_d1'].get_shape().as_list()[0]])\n    dense_1 = dense(flat, weights['W_d1'], biases['b_d1'])\n    dense_2 = dense(dense_1, weights['W_d2'], biases['b_d2'])\n    dense_2 = tf.nn.dropout(dense_2, 1-den_dropout)\n    \n    # output layer: \n    out_layer_z = tf.add(tf.matmul(dense_2, weights['W_out']), biases['b_out'])\n    #print(out_layer_z.shape)\n    return out_layer_z\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a271005688c9da3202d9e1a0fd942d9296bc78a"},"cell_type":"code","source":"bias_dict = {\n    'b_c1': tf.Variable(tf.zeros([n_conv_1])),\n    'b_c2': tf.Variable(tf.zeros([n_conv_1])),\n    'b_c3': tf.Variable(tf.zeros([n_conv_1])),\n    'b_c4': tf.Variable(tf.zeros([n_conv_1])),\n    'b_c5': tf.Variable(tf.zeros([n_conv_2])),\n    'b_c6': tf.Variable(tf.zeros([n_conv_2])),\n    'b_c7': tf.Variable(tf.zeros([n_conv_2])),\n    'b_c8': tf.Variable(tf.zeros([n_conv_2])),\n    #'b_c9': tf.Variable(tf.zeros([n_conv_3])),\n    #'b_c10': tf.Variable(tf.zeros([n_conv_3])),\n    #'b_c11': tf.Variable(tf.zeros([n_conv_3])),\n    #'b_c12': tf.Variable(tf.zeros([n_conv_3])),\n    'b_d1': tf.Variable(tf.zeros([n_dense])),\n    'b_d2': tf.Variable(tf.zeros([n_dense2])),\n    'b_out': tf.Variable(tf.zeros([n_classes]))\n}\n\n# calculate number of inputs to dense layer: \nfull_square_length = np.sqrt(n_input)\npooled_square_length = int(full_square_length / (pool_size*pool_size))\ndense_inputs = pooled_square_length**2 * n_conv_2\n\nweight_dict = {\n    'W_c1': tf.get_variable('W_c1',  [k_conv_1, k_conv_1, 1, n_conv_1], initializer=wt_init),\n    'W_c2': tf.get_variable('W_c2',  [k_conv_1, k_conv_1, n_conv_1, n_conv_1], initializer=wt_init),\n    'W_c3': tf.get_variable('W_c3',  [k_conv_1, k_conv_1, n_conv_1, n_conv_1], initializer=wt_init),\n    'W_c4': tf.get_variable('W_c4',  [k_conv_1, k_conv_1, n_conv_1, n_conv_1], initializer=wt_init),\n    'W_c5': tf.get_variable('W_c5',  [k_conv_2, k_conv_2, n_conv_1, n_conv_2], initializer=wt_init),\n    'W_c6': tf.get_variable('W_c6',  [k_conv_2, k_conv_2, n_conv_2, n_conv_2], initializer=wt_init),\n    'W_c7': tf.get_variable('W_c7',  [k_conv_2, k_conv_2, n_conv_2, n_conv_2], initializer=wt_init),\n    'W_c8': tf.get_variable('W_c8',  [k_conv_2, k_conv_2, n_conv_2, n_conv_2], initializer=wt_init),\n    #'W_c9': tf.get_variable('W_c9',  [k_conv_3, k_conv_3, n_conv_2, n_conv_3], initializer=wt_init),\n    #'W_c10': tf.get_variable('W_c10',  [k_conv_3, k_conv_3, n_conv_3, n_conv_3], initializer=wt_init),\n    #'W_c11': tf.get_variable('W_c11',  [k_conv_3, k_conv_3, n_conv_3, n_conv_3], initializer=wt_init),\n    #'W_c12': tf.get_variable('W_c12',  [k_conv_3, k_conv_3, n_conv_3, n_conv_3], initializer=wt_init),\n    'W_d1': tf.get_variable('W_d1',  [dense_inputs, n_dense], initializer=wt_init),\n    'W_d2': tf.get_variable('W_d2',  [n_dense, n_dense2], initializer=wt_init),\n    'W_out': tf.get_variable('W_out',  [n_dense2, n_classes], initializer=wt_init)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f46e56776c0ece31c60657ef40711dee50812cca"},"cell_type":"code","source":"predictions = network(x, weight_dict, bias_dict, n_input, \n                      pool_size, mp_dropout, dense_dropout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6e9ee0073ac97192d553a01fc9eba6fcaa71804"},"cell_type":"code","source":"print(predictions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a58508bf43a71972ab877f46a675f9396b5190b8"},"cell_type":"code","source":"print(y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e222805135809244077beb4e8f4e3e6ed085dcbc"},"cell_type":"code","source":"cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=predictions ,labels=y))\noptimizer=tf.train.AdamOptimizer().minimize(cost)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b3ee544d805125a1dad026af3bc821fe98a5094"},"cell_type":"code","source":"correct_prediction=tf.equal(tf.argmax(predictions,1),tf.argmax(y,1))\naccuracy_pct=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a922f3f20612df71055c6fc24f7bfebe41d611ab"},"cell_type":"code","source":"init=tf.global_variables_initializer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93f7c2f1c0407a7bf30cefe26dc1dc376b6789af"},"cell_type":"code","source":"predict=tf.argmax(predictions,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d76b2e4046f9fddbbdc013203299afe860628f51"},"cell_type":"code","source":"#x_train.shape[0]\n#x_train.shape,y_train.shape\n#keep_prob = tf.placeholder('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a9cf36eea9e303a2f648be9346078ab602fb306"},"cell_type":"code","source":"#batch_start_idx = (1 * batch_size) % (x_train.shape[0] - batch_size)\n#batch_end_idx = batch_start_idx + batch_size\n#batch_X = x_train[batch_start_idx:batch_end_idx]\n#batch_Y = y_train[batch_start_idx:batch_end_idx]\n#batch_X.shape,batch_Y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"464cd185dbe7f1e8141f0cb72d0e3e47555b5def","scrolled":true},"cell_type":"code","source":"with tf.Session() as session:\n    session.run(init)\n    \n    print(\"Training for\", epochs, \"epochs.\")\n    \n    # loop over epochs: \n    for epoch in range(epochs):\n        \n        avg_cost = 0.0 # track cost to monitor performance during training\n        avg_accuracy_pct = 0.0\n        \n        # loop over all batches of the epoch:\n        n_batches = int(x_train.shape[0] / batch_size)\n        #batchnumber=0\n        for i in range(n_batches):\n            \n            # batch_x, batch_y = mnist.train.next_batch(batch_size)\n            #batchnumber= batchnumber+1\n            batch_start_idx = (i * batch_size) % (x_train.shape[0] - batch_size)\n            batch_end_idx = batch_start_idx + batch_size\n            batch_X = x_train[batch_start_idx:batch_end_idx]\n            batch_Y = y_train[batch_start_idx:batch_end_idx]\n            \n            # feed batch data to run optimization and fetching cost and accuracy: \n            _, batch_cost, batch_acc, Predict = session.run([optimizer, cost, accuracy_pct, predictions], \n                                                   feed_dict={x: batch_X, y: batch_Y})\n            \n            # accumulate mean loss and accuracy over epoch: \n            avg_cost += batch_cost / n_batches\n            avg_accuracy_pct += batch_acc / n_batches\n            \n        # output logs at end of each epoch of training:\n        print(\"Epoch \", '%03d' % (epoch+1), \n              \": cost = \", '{:.3f}'.format(avg_cost), \n              \", accuracy = \", '{:.2f}'.format(avg_accuracy_pct), \"%\", \n              sep='')\n    \n    print(\"Training Complete. Testing Model.\\n\")\n    \n    test_cost = cost.eval({x: x_test, y: y_test})\n    test_accuracy_pct = accuracy_pct.eval({x: x_test, y: y_test})\n    \n    print(\"Test Cost:\", '{:.3f}'.format(test_cost))\n    print(\"Test Accuracy: \", '{:.2f}'.format(test_accuracy_pct), \"%\", sep='')\n    \n    #predicted_lables = predict1.eval({x: x_train})\n    #print(len(predicted_lables))\n    predicted_lables = np.zeros(x_test_data.shape[0])\n    for i in range(0,x_test_data.shape[0]//batch_size):\n        predicted_lables[i*batch_size : (i+1)*batch_size] = predict.eval({x: x_test_data[i*batch_size : (i+1)*batch_size], \n                                                                                })\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b254d5a23e97d92c234295b1a11c9eac2bd3b6af"},"cell_type":"code","source":"predicted_lables.shape,len(predicted_lables)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba97242db6d75caffbbbf8a06717c9e1ccb35ae1"},"cell_type":"code","source":"np.savetxt('Avinash1.csv', \n                        np.c_[range(1,len(x_test_data)+1),predicted_lables], \n                        delimiter=',', \n                        header = 'ImageId,Label', \n                        comments = '', \n                        fmt='%d')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a4aa957db12f895e33c117d882353423f1849ad"},"cell_type":"code","source":"predicted_lables[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"243518d3573f0b94fb7ea6a2747d369fc1311978"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}