{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ab8725a3-695d-0783-6ed2-c3d4c3877adf" }, "outputs": [], "source": [ "import time\n", "import numpy as np # linear algebra\n", "import tensorflow as tf # for a neural network...?\n", "\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "#from subprocess import check_output\n", "#print(check_output([\"dir\", \"..\\input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cdb3f59a-6409-2957-b241-a2145f23a0cd" }, "outputs": [], "source": [ "# Constants that will probably help\n", "N_CLASSES = 10\n", "N_FEATURES = 28*28\n", "BATCH_SIZE = 50\n", "MIN_AFTER = 10000\n", "CAPACITY = MIN_AFTER + (3*BATCH_SIZE)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e4ec0628-8c65-7afc-c9b7-7d860d470cfe" }, "outputs": [], "source": [ "# Reset the graph\n", "tf.reset_default_graph()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "67aa49b4-6896-1ef8-7c3a-e1248d421bfb" }, "outputs": [], "source": [ "# Reading in data as an input pipeline\n", "# 1) List of file names (not going to do file shuffling or put an epoch limit)\n", "# a) going to put in both the names for the training and test data\n", "file_names = [\"../input/train.csv\"]#, \"../input/test.csv\"]\n", "\n", "# 2) Filename queue (this is for passing in the files)\n", "# a) a queue can be made from tf.train.string_input_producer\n", "file_queue = tf.train.string_input_producer(file_names, num_epochs=1)\n", "\n", "\n", "# 3) Creating a reader for the specific file format\n", "# a) the tf.TextLineReader will output the lines in a file!\n", "# b) have it read one file to pass into the decoder\n", "# i) I think key = name, value = file information?\n", "file_reader = tf.TextLineReader(skip_header_lines=1, name=\"Training\")\n", "key, value = file_reader.read(file_queue)\n", "\n", "\n", "# 4) Creating a decoder to grab the information\n", "# a) use the csv decoder since the file is a csv file\n", "# b) returns multidimensional array with each feature making up a column\n", "# c) requires record_defaults: a Tensor of #features size, with a scalar\n", "# representing the datatype of that feature\n", "# d) separate the features and labels\n", "record_defaults = [[1]]*(N_FEATURES+1) # +1 for the labels column\n", "all_data = tf.decode_csv(value, record_defaults=record_defaults)\n", "\n", "outputs = all_data[0]\n", "features = tf.stack(all_data[1:])\n", "\n", "# 5) Creating batches from the information\n", "feature_batch, output_batch = tf.train.shuffle_batch([features,outputs],\n", " BATCH_SIZE,\n", " CAPACITY,\n", " MIN_AFTER)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e981cd0f-eb85-3d77-7d33-3f10918c2825" }, "outputs": [], "source": [ "# # Now preparing the softmax neural network - gives 0.892 accuracy\n", "# # 1) Creating placeholders for the data\n", "# X = tf.placeholder(tf.float32, shape=[None, N_FEATURES])\n", "# Y = tf.placeholder(tf.float32, shape=[None, N_CLASSES])\n", "\n", "# #2) Create softmax variables to manipulate the data\n", "# weights = tf.Variable(tf.zeros([N_FEATURES, N_CLASSES]))\n", "# biases = tf.Variable(tf.zeros([N_CLASSES]))\n", "\n", "# # 3) Going to do class predictions and loss function\n", "# # a) for now, the prediction function is x*weights + biases\n", "# # b) the loss is the mean of the predictions that were wrong\n", "# # i) logits are the predictions\n", "# Yhat = tf.matmul(X, weights) + biases\n", "\n", "# cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n", " # labels=Y, logits=Yhat))\n", "\n", "# # 4) Create a way to train the model, using the GradientDescentOptimizer\n", "# # a) can also use AdadeltaOptimizer, an adaptive learning optimizer\n", "# training_step = tf.train.GradientDescentOptimizer(\n", " # learning_rate=0.5).minimize(cross_entropy)\n", "\n", "# # 5) Evaluate the model\n", "# correct_prediction = tf.equal(tf.argmax(Yhat,1), tf.argmax(Y,1))\n", "# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8df840b4-44a8-c501-0d81-d632b1da64b5" }, "outputs": [], "source": [ "# Now preparing for a Convolutional Network - gives 0.96 accuracy!\n", "# 0) Creating placeholders for the data\n", "X = tf.placeholder(tf.float32, shape=[None, N_FEATURES])\n", "Y = tf.placeholder(tf.float32, shape=[None, N_CLASSES])\n", "\n", "# 1) Creating a weight-initialization function\n", "def weight_variable(shape):\n", " initial = tf.truncated_normal(shape, stddev=0.1)\n", " return tf.Variable(initial)\n", "\n", "# 2) Creating a bias-initialization function\n", "def bias_variable(shape):\n", " initial = tf.constant(0.1, shape=shape)\n", " return tf.Variable(initial)\n", "\n", "# 3) Creating the convolution filter\n", "def convolution_2d(X, weights, strides=[1,1,1,1], padding=\"SAME\"):\n", " # for now, don't do anything\n", " return tf.nn.conv2d(X, weights, strides=strides, padding=padding)\n", "\n", "# 4) Creating the pooling operation\n", "def max_pool_2x2(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], \n", " padding='SAME'):\n", " return tf.nn.max_pool(X, ksize=ksize, strides=strides, \n", " padding=padding)\n", "\n", "# 5) Creating the actual convolutional layers\n", "W_conv1 = weight_variable([5, 5, 1, 32])\n", "b_conv1 = bias_variable([32])\n", "image = tf.reshape(X, [-1,28,28,1])\n", "h_conv1 = tf.nn.relu(convolution_2d(image, W_conv1) + b_conv1)\n", "h_pool1 = max_pool_2x2(h_conv1)\n", "\n", "W_conv2 = weight_variable([5, 5, 32, 64])\n", "b_conv2 = bias_variable([64])\n", "h_conv2 = tf.nn.relu(convolution_2d(h_pool1, W_conv2) + b_conv2)\n", "h_pool2 = max_pool_2x2(h_conv2)\n", "\n", "# 6) Create the fully connected layer\n", "W_fullc = weight_variable([7 * 7 * 64, 1024])\n", "b_fullc = bias_variable([1024])\n", "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n", "h_fullc = tf.nn.relu(tf.matmul(h_pool2_flat, W_fullc) + b_fullc)\n", "\n", "# 7) Create something that can do dropout to prevent overfitting\n", "prob_keep = tf.placeholder(tf.float32)\n", "h_fullc_drop = tf.nn.dropout(h_fullc, prob_keep)\n", "\n", "# 8) Create the output layer and calculate cross extropy\n", "W_out = weight_variable([1024, 10])\n", "b_out = bias_variable([10])\n", "Yhat = tf.matmul(h_fullc_drop, W_out) + b_out\n", "\n", "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n", " labels=Y, logits=Yhat))\n", "\n", "# 9) Create a training optimizer - using Adam\n", "training_step = tf.train.AdamOptimizer(\n", " learning_rate=1e-4).minimize(cross_entropy)\n", "\n", "# 10) Evaluate the model\n", "correct_prediction = tf.equal(tf.argmax(Yhat,1), tf.argmax(Y,1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7323a69e-4fb6-70c6-f321-59199f028fe6" }, "outputs": [], "source": [ "# Add ops to save and restore all the variables.\n", "saver = tf.train.Saver()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9b522515-879c-733e-3232-5a94a45282e5" }, "outputs": [], "source": [ "save_path = ''\n", "# Running the session!\n", "with tf.Session() as sess:\n", " begin = time.time()\n", " sess.run(tf.global_variables_initializer())\n", " after_global_initialize = time.time()\n", " sess.run(tf.local_variables_initializer())\n", " after_local_initialize = time.time()\n", " \n", " # 1) Populate the file_queue\n", " # a) requires you to initialize a Coordinator\n", " coord = tf.train.Coordinator()\n", " after_coord_initialize = time.time()\n", " threads = tf.train.start_queue_runners(sess=sess,coord=coord)\n", " after_thread_initialize = time.time()\n", " \n", " validation_x = []\n", " validation_y = []\n", " \n", " # getting a bunch of validation data\n", " # the 126 is just a number I picked; can be pretty much anything\n", " # 126 = 840*0.15, where 840 is the number of batches\n", " #validation_time_counter = time.time()\n", " for i in range(126):\n", " validation_batch = sess.run([feature_batch, output_batch])\n", " validation_x.extend(validation_batch[0])\n", " validation_y.extend(validation_batch[1])\n", " #print(\"Validation\", i, time.time() - validation_time_counter)\n", " #validation_time_counter = time.time()\n", " after_create_validation = time.time()\n", " counter = 0\n", " #training_time_counter = time.time()\n", " try:\n", " while not coord.should_stop():\n", " # example = one observation; label = the label\n", " example_batch, label_batch = sess.run([feature_batch, \n", " output_batch])\n", " feed_dict = {X:example_batch, Y:tf.one_hot(label_batch,\n", " depth=N_CLASSES).eval(),\n", " prob_keep:0.5}\n", " sess.run(training_step, feed_dict=feed_dict)\n", " counter += 1\n", " #print(\"Training\", counter, time.time() - training_time_counter)\n", " #training_time_counter = time.time()\n", " except tf.errors.OutOfRangeError:\n", " print(\"Finished training!\")\n", " finally:\n", " # close the Coordinate since we don't need it anymore\n", " coord.request_stop()\n", " after_training = time.time()\n", " print(counter)\n", " \n", " print(\"Validation accuracy:\", \n", " accuracy.eval(feed_dict={X:validation_batch[0], \n", " Y:tf.one_hot(validation_batch[1], \n", " depth=N_CLASSES).eval(),\n", " prob_keep:1.0}))\n", " after_evaluation = time.time()\n", " \n", " save_path = saver.save(sess, \"my-model\")\n", " print(\"Model saved in file: %s\" % save_path)\n", " \n", " print(\"Time for Global Initalize:\", after_global_initialize - begin)\n", " print(\"Time for Local Initalize:\", after_local_initialize - after_global_initialize)\n", " print(\"Time for Coord Initalize:\", after_coord_initialize - after_local_initialize)\n", " print(\"Time for Thread Initalize:\", after_thread_initialize - after_coord_initialize)\n", " print(\"Time for Creating Validation:\", after_create_validation - after_thread_initialize)\n", " print(\"Time for Training:\", after_training - after_create_validation)\n", " print(\"Time for Evaluation:\", after_evaluation - after_training)\n", " coord.join(threads)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bf2fd4c1-1f1a-b02e-29c8-cdcb593e934b" }, "outputs": [], "source": [ "# Want to get output for the test data - need to repeat some things!\n", "file_queue_test = tf.train.string_input_producer([\"../input/test.csv\"], \n", " num_epochs=1,\n", " name=\"Test\")\n", "\n", "file_reader_test = tf.TextLineReader(skip_header_lines=1, name=\"Testing\")\n", "key_test, value_test = file_reader_test.read(file_queue_test)\n", "\n", "record_defaults_test = [[1]]*(N_FEATURES)\n", "all_data_test = tf.decode_csv(value_test,\n", " record_defaults=record_defaults_test)\n", "\n", "X_test = tf.placeholder(tf.float32, shape=[None, N_FEATURES])\n", "\n", "# Calculate prediction step\n", "image_test = tf.reshape(X_test, [-1,28,28,1])\n", "h_conv1_test = tf.nn.relu(convolution_2d(image_test, W_conv1) + b_conv1)\n", "h_pool1_test = max_pool_2x2(h_conv1_test)\n", "h_conv2_test = tf.nn.relu(convolution_2d(h_pool1_test,W_conv2)+b_conv2)\n", "h_pool2_test = max_pool_2x2(h_conv2_test)\n", "h_pool2_flat_test = tf.reshape(h_pool2_test, [-1, 7*7*64])\n", "h_fullc_test = tf.nn.relu(tf.matmul(h_pool2_flat_test, W_fullc) +b_fullc)\n", "Ypred = tf.matmul(h_fullc_test, W_out) + b_out" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d8d21d33-5973-24ee-1264-c5d1cfb41bcc" }, "outputs": [], "source": [ "predictions = []\n", "with tf.Session() as sess:\n", " saver.restore(sess, \"model\")\n", " print(\"Model restored.\")\n", " \n", " sess.run(tf.global_variables_initializer())\n", " sess.run(tf.local_variables_initializer())\n", " \n", " # 1) Populate the file_queue\n", " # a) requires you to initialize a Coordinator\n", " coord = tf.train.Coordinator()\n", " threads = tf.train.start_queue_runners(sess=sess,coord=coord)\n", " \n", " test_data = []\n", " counter = 0\n", " try:\n", " while not coord.should_stop():\n", " # example = one observation; label = the label\n", " test_data.append(sess.run(all_data_test))\n", " counter += 1\n", " #print(\"Training\", counter, time.time() - training_time_counter)\n", " #training_time_counter = time.time()\n", " except tf.errors.OutOfRangeError:\n", " print(\"Finished training!\", counter)\n", " finally:\n", " # close the Coordinate since we don't need it anymore\n", " coord.request_stop()\n", " \n", " \n", " predictions = sess.run(Ypred, feed_dict={X_test:test_data})\n", " \n", " with open(\"prediction.txt\", 'w') as file:\n", " file.write(predictions)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a3deb98a-5354-5ac9-475b-f0d9d25a9348" }, "outputs": [], "source": [ "print(predictions)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "34550260-6261-27fd-8d4a-9079815c997c" }, "outputs": [], "source": "" } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }