{"cells":[{"metadata":{"id":"7uwKBcHblXfP","colab_type":"code","colab":{},"trusted":true,"_uuid":"215ad5f994785790c82e35dd65aa728d2b00f887"},"cell_type":"code","source":"import numpy as np\nfrom keras.datasets import mnist\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import Flatten\nfrom keras.layers.convolutional import Conv2D\nfrom keras.layers.convolutional import MaxPooling2D\nfrom keras.layers.convolutional import AveragePooling2D\nfrom keras.layers import Activation\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import np_utils\nfrom keras.models import load_model\nfrom keras import optimizers\nfrom sklearn.metrics import confusion_matrix\nimport pandas as pd\nimport seaborn as sns\nimport json\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nplt.style.use('ggplot')","execution_count":null,"outputs":[]},{"metadata":{"id":"7Co2-jweliAR","colab_type":"code","colab":{},"trusted":true,"_uuid":"988bd7d80021bdb646248590ad7097674ca95592"},"cell_type":"code","source":"from numpy import genfromtxt\n\ntrain = genfromtxt(\"../input/train.csv\",delimiter=',', skip_header=1)\ntest = genfromtxt(\"../input/test.csv\",delimiter=',', skip_header=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"DvOdxto7v6RJ","colab_type":"code","outputId":"57e620d4-8861-40e8-da5e-b06d018b94e9","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":true,"_uuid":"ded8a3c6cea2e911e4461d040a1e81718171feca"},"cell_type":"code","source":"X = train[:,1:]\ny = train[:,0]\n\nprint(\"X_train shape: {}\".format(X.shape))\nprint(\"y_train shape: {}\".format(y.shape))\nprint(\"X_test shape: {}\".format(test.shape))\n\nprint(\"Classes: {}\".format(np.unique(y)))\n\nnum_classes = len(np.unique(y))\nprint(\"Number of classes: {}\".format(num_classes))\n","execution_count":null,"outputs":[]},{"metadata":{"id":"g8ZIdl2Vwn7d","colab_type":"code","colab":{},"trusted":true,"_uuid":"887c2392baf20f8e2b0fb003d8efa4dedba52b72"},"cell_type":"code","source":"X_reshaped = X.reshape((X.shape[0],28,28,1))\nX_test_reshaped = test.reshape((test.shape[0],28,28,1))\n\n# normalize inputs from 0-255 to 0-1\nX_reshaped = X_reshaped/ 255\nX_test_reshaped = X_test_reshaped / 255\n\n# one hot encode outputs\ny_oh = np_utils.to_categorical(y)\n\nnum_classes = y_oh.shape[1]\n","execution_count":null,"outputs":[]},{"metadata":{"id":"7CrTh5yRwGhE","colab_type":"code","colab":{},"trusted":true,"_uuid":"ffe8dace14fae56f0f4538fe49636b13e51b2a5f"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_reshaped, y_oh, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"id":"PmBCh2Szk0LJ","colab_type":"text","_uuid":"0fb684a3b1d061c093e6c6fb024e70d23d6e8c67"},"cell_type":"markdown","source":"**Objective**\nThis is a classification task which takes as input gray scale images of hand-written numbers and predicts which of 10 numbers it shows. \n\nThe input images are 28 x 28 pixels (784 features) while the output variable gives the ground-truth label of the input images and takes values between 0 - 10. \n\nA classifier utilizing CNN will be trained on (X_train, y_train). (X_test, y_test) will be used as the validation set.\n\n**Data preprocessing**\nTo apply CNNs, the data will be reshaped into 4D-arrays of (m, 28, 28, 1), where m denotes the number of samples in each data set. Furthermore, all  inputs will be normalized by 255 (the max intensity value) to minimize the effect of varying image intensity across images.\n\nOne-hot encoding will be applied to y labels"},{"metadata":{"id":"vJ4wHPWfwwAe","colab_type":"code","outputId":"82d41615-7683-44c8-f9f9-5f7e5d928d27","colab":{"base_uri":"https://localhost:8080/","height":282},"trusted":true,"_uuid":"83493a810800f922e743795be86075acabd6cc16"},"cell_type":"code","source":"#visualize first 9 images in training set\n\nfig, ax = plt.subplots(3, 3, sharex='col', sharey='row', figsize=(4,4))\nfig.subplots_adjust(hspace=0.5)\nindex = 0\n\nfor i in range(3):\n    for j in range(3):\n        ax[i, j].imshow(X_train[index,:,:,0], cmap=plt.get_cmap('gray'))\n        ax[i, j].set_title(\"Label: {}\".format(np.argmax(y[index])))\n        index +=1\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"eCLO93Eww1XH","colab_type":"text","_uuid":"9659ceea6f6e60d0f7b87270c2e3dabaefecc83a"},"cell_type":"markdown","source":"**LeNet Architecture**\n\nA model based on the LeNet CNN architecture was trained using Adam optimizer. "},{"metadata":{"id":"yBD0RKDY5QKR","colab_type":"code","colab":{},"trusted":true,"_uuid":"b8e2464b8e69cd6ae5dfd489b0a686fb607e26d9"},"cell_type":"code","source":"def create_lenet():\n  # LeNet\n\n  seed = 7\n  np.random.seed(seed)\n\n\n  # build the model\n  # create Lenet model. Use same padding for 1st layer so that output size remains at 28 x 28 similar to Lenet output layer 1\n  lenet_model = Sequential()\n  lenet_model.add(Conv2D(6, kernel_size = (5, 5), strides = (1,1), input_shape=(28,28,1), padding = \"same\", activation='tanh'))\n  lenet_model.add(AveragePooling2D(pool_size=(2, 2), strides = (2,2), padding = 'valid'))\n\n  lenet_model.add(Conv2D(16, kernel_size = (5, 5), strides = (1,1), padding = \"valid\", activation='tanh'))\n  lenet_model.add(AveragePooling2D(pool_size=(2, 2), strides = (2,2), padding = 'valid'))\n\n  lenet_model.add(Conv2D(120, kernel_size = (5, 5), strides = (1,1), padding = \"valid\", activation='tanh'))\n\n  lenet_model.add(Flatten())\n  lenet_model.add(Dense(84, activation='tanh'))\n  lenet_model.add(Dense(num_classes, activation='softmax'))\n  \n  return lenet_model\n","execution_count":null,"outputs":[]},{"metadata":{"id":"0xQD14rul4il","colab_type":"code","outputId":"3009c19f-a6e1-488b-f6e4-a989a10cf138","colab":{"base_uri":"https://localhost:8080/","height":1054},"trusted":true,"_uuid":"44df7edf8bff38b588703300fac3d5cf9f008123"},"cell_type":"code","source":"lenet_model= create_lenet()\n\n# Compile model\nlenet_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n# Fit the model\nlenet_model_history = lenet_model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=30, \n                                      batch_size=128, verbose=2, shuffle=True)\n          ","execution_count":null,"outputs":[]},{"metadata":{"id":"jb55hZn51MGe","colab_type":"code","colab":{},"trusted":true,"_uuid":"1cba2f6408c76987a192cb60f5b1de7b3fbcd06a"},"cell_type":"code","source":"# evaluates model by comparing training and validation accuracy and losses\n\ndef model_evaluate(history):\n\n  f, (ax1,ax2) = plt.subplots(2,1,sharex=True, figsize = (8,8))\n  f.subplots_adjust(hspace=0.3)\n  \n  ax1.plot(history.history['acc'], 'r', linewidth=1)\n  ax1.plot(history.history['val_acc'], 'b', linewidth=1)\n\n  # Plot legend and use the best location automatically: loc = 0.\n  ax1.legend([\"Train Acc.\", \"Validation Acc.\"], loc = 0)\n  ax1.set_title(\"Training/Validation Acc. per Epoch\")\n  ax1.set_ylabel(\"Accuracy\")\n     \n  text = \"Final Training Accuracy:{:.2f}%\\nFinal Val. Accuracy:{:.2f}%\".format(history.history['acc'][-1]*100,\n                                                                            history.history['val_acc'][-1]*100)\n  ax1.text(0.5, 0.5,text, transform=ax1.transAxes, fontsize=12)\n  \n  ax2.plot(history.history['loss'], 'r', linewidth=1)\n  ax2.plot(history.history['val_loss'], 'b', linewidth=1)\n  \n  # Plot legend and use the best location automatically: loc = 0.\n  ax2.legend([\"Train loss\", \"Validation loss\"], loc = 0)\n  ax2.set_title(\"Training/Validation loss per Epoch\")\n  ax2.set_xlabel(\"Epoch\")\n  ax2.set_ylabel(\"Loss\")\n  \n  plt.show()\n  \n  ","execution_count":null,"outputs":[]},{"metadata":{"id":"KmtBjR50a2l2","colab_type":"code","colab":{},"trusted":true,"_uuid":"fb8e2f87b306e8ed1ea5716b662411d9f52e6a20"},"cell_type":"code","source":"#save trained model and model history\ndef save_files(model, model_name, model_history):\n  model.save(model_name+\".h5\")\n  files.download(model_name+\".h5\")\n  \n  with open(model_name+\".json\", 'w') as f:\n    json.dump(model_history.history, f)\n  files.download(model_name+\".json\")","execution_count":null,"outputs":[]},{"metadata":{"id":"1js7A8nfdr1p","colab_type":"code","outputId":"b69e228a-3728-4546-f937-8a564bb19e96","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true,"_uuid":"43d36b26173ecf005944a403f4b3ab66b936b46c"},"cell_type":"code","source":"model_evaluate(lenet_model_history)\n#save_files(lenet_model, \"lenet_model\", lenet_model_history)","execution_count":null,"outputs":[]},{"metadata":{"id":"sDROgr9ypcZQ","colab_type":"text","_uuid":"fd2c40af2945ac237f05975d216b2695893345c0"},"cell_type":"markdown","source":"For the LeNet model, batch sizes of 128 and 256 were tried but it was found that the results were similar. The result above shows the trained model using a batch size of 128.\n\nThe validation accuracy is 97.9%, 2% lower than the traiing accuracy.  This suggests that the model may be slightly overfitting on the training set. \n\nTo mitigate the overfitting on the LeNet model, three options were explored:\n1) Add dropout layers to the mode\n2) Use data augmentation to increase the size of the training set\n3) Explore different CNN architectures"},{"metadata":{"id":"g11JF2YZ1YRl","colab_type":"text","_uuid":"80631ca21fdc71940e8df8602f5871ef342fcfc2"},"cell_type":"markdown","source":"**1) Add dropout layers and use ReLu Activation**\n\nTo minimize the overfitting, dropout layers were added to the Lenet architecture. ReLu activation was used instead of tanh"},{"metadata":{"id":"c2Vw7H5jGL0P","colab_type":"code","outputId":"6b92d77c-c11b-4580-ca68-80ef092ef904","colab":{"base_uri":"https://localhost:8080/","height":1054},"trusted":true,"_uuid":"64566c7b8f9548d0d18acaa2ccbf72a42be5221a"},"cell_type":"code","source":"def create_lenet_dropout():\n\n  seed = 7\n  np.random.seed(seed)\n\n\n  # build the model\n  # create Lenet model. Use same padding for 1st layer so that output size remains at 28 x 28 similar to Lenet output layer 1\n  lenet_model = Sequential()\n  lenet_model.add(Conv2D(6, kernel_size = (5, 5), strides = (1,1), input_shape=(28,28,1), padding = \"same\", activation='relu'))\n  lenet_model.add(AveragePooling2D(pool_size=(2, 2), strides = (2,2), padding = 'valid'))\n\n  \n  lenet_model.add(Conv2D(16, kernel_size = (5, 5), strides = (1,1), padding = \"valid\", activation='relu'))\n  lenet_model.add(AveragePooling2D(pool_size=(2, 2), strides = (2,2), padding = 'valid'))\n  lenet_model.add(Dropout(0.2))\n  \n  lenet_model.add(Conv2D(120, kernel_size = (5, 5), strides = (1,1), padding = \"valid\", activation='relu'))\n  \n  lenet_model.add(Flatten())\n  lenet_model.add(Dropout(0.2))\n  lenet_model.add(Dense(84, activation='relu'))\n  lenet_model.add(Dense(num_classes, activation='softmax'))\n  \n  return lenet_model\n\n\nlenet_dropout_model= create_lenet_dropout()\n\n# Compile model\nlenet_dropout_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n\n# Fit the model\nlenet_dropout_model_history = lenet_dropout_model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=30, batch_size=128, verbose=2, shuffle=True)\n          \n","execution_count":null,"outputs":[]},{"metadata":{"id":"NbP-yPmhGkcB","colab_type":"code","outputId":"2965aad7-b7d4-401d-a1af-b9961e5e0f51","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true,"_uuid":"c1719c268e3150f30769dbe8ccc57832edd2dfb3"},"cell_type":"code","source":"model_evaluate(lenet_dropout_model_history)\n#save_files(lenet_dropout_model, \"lenet_dropout_model\", lenet_dropout_model_history)","execution_count":null,"outputs":[]},{"metadata":{"id":"zLcl7gfi_eiw","colab_type":"text","_uuid":"f069df3687cb4a763d5bd383cd0cff80c2caaed9"},"cell_type":"markdown","source":"With dropout, the validation accuracy improved to 98.8%, which suggests that this model generalizes better. "},{"metadata":{"id":"fzcZ3uoD4OJa","colab_type":"text","_uuid":"0a8e2acfef5e1e4812f78ec25308cd8d98b306f1"},"cell_type":"markdown","source":"**2) LeNet with data augmentation**\nTo reduce overfitting, Keras ImageDataGenerator was used to sample training data and augment them, effectively giving a large pool of training samples to train the base LeNet model"},{"metadata":{"id":"ZVSmqNcI5l3X","colab_type":"code","colab":{},"trusted":true,"_uuid":"6dca45f014409919e29854f75c2008c2b5d36225"},"cell_type":"code","source":"lenet_model_dataAug= create_lenet()\n\n# Compile model\nlenet_model_dataAug.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"id":"WXX3wYlPrIdJ","colab_type":"code","outputId":"5ee89df0-e4b3-4dbf-9e13-a18295a16cc3","colab":{"base_uri":"https://localhost:8080/","height":1037},"trusted":true,"_uuid":"71b9e7dd4cebacdccb344bd102316cc43a674eae"},"cell_type":"code","source":"# create image generator to create augmented data from X_train\ngen = ImageDataGenerator(rotation_range=5, width_shift_range=0.05, shear_range=0.2,\n                         height_shift_range=0.05, zoom_range=0.05)\n\n\ntrain_generator = gen.flow(X_train, y_train, batch_size=128)\n\nlenet_model_dataAug_history = lenet_model_dataAug.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=30, validation_data=(X_val, y_val), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"id":"2sn_7AUBs35W","colab_type":"code","outputId":"e1397d7f-4ae5-42b6-af98-32608b63d635","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true,"_uuid":"353e4539519f6f7ce305ce56b3c340aeed36dd0e"},"cell_type":"code","source":"model_evaluate(lenet_model_dataAug_history)\n#save_files(lenet_model_dataAug, \"lenet_model_dataAug\", lenet_model_dataAug_history)","execution_count":null,"outputs":[]},{"metadata":{"id":"rkVK-MICFpla","colab_type":"text","_uuid":"3bbc968474f571f3436cf7d9e0fd41e77dc201f0"},"cell_type":"markdown","source":"With data augmentation (i.e. a larger training set), the base LeNet model does generalize better to the validation data,  with an improvement in validation accuracy to 98.8%, comparable to the model with dropout.\n\n"},{"metadata":{"id":"ZFo1g_01RJUS","colab_type":"text","_uuid":"c32af8843a08a3f98f7b1945c9eb18404ea81b4d"},"cell_type":"markdown","source":"**3) Experiments with other CNN Architectures:**\nTo see if further improvements in prediction accuracy could be made, experiments were conducted using different model architectures based loosely on AlexNet and VGGNet"},{"metadata":{"id":"4wvrGOOYzIEs","colab_type":"text","_uuid":"0719b3f0bed4daac0b2137d30efd4df2129ed90d"},"cell_type":"markdown","source":"**a) CNN with 4 Conv Layers + 3 FCs** \n\nAlexNet has 5 Conv Layers + 3 FCs and takes in images of 227x227 pixels to give 1000 possible output classes. \n\nThe inputs to the 2nd Conv layer in AlexNet are 27 x 27 pixels, similar in size to the MNIST data (28 x 28) and the MNIST data set is less complex (only 10 classes). A model architecture was therefore adapted loosely from AlexNet with the first Conv layer removed leaving 4 Conv Layers + 3 FCs \n\nDue to the relative simplicity of the MNIST data, the number of filters was also reduced to keep the number of parameters low without compromising prediction accuracy.  "},{"metadata":{"id":"mAkTaPJr6Dcy","colab_type":"code","colab":{},"trusted":true,"_uuid":"5d2eaca774d057aaa0341efc6c4687d5f4dbd315"},"cell_type":"code","source":"def create_model_4Conv_3FC():\n  seed = 7\n  np.random.seed(seed)\n  \n  model = Sequential()\n  model.add(Conv2D(12, kernel_size = (5, 5), strides = (1,1), input_shape=(28,28,1), padding=\"same\", activation='relu'))\n  model.add(MaxPooling2D(pool_size=(3, 3), strides = (2,2)))\n  model.add(BatchNormalization())\n\n  model.add(Conv2D(48, kernel_size = (3, 3), strides = (1,1), padding = \"same\", activation='relu'))\n  model.add(Conv2D(48, kernel_size = (3, 3), strides = (1,1), padding = \"same\", activation='relu'))\n  model.add(Conv2D(32, kernel_size = (3, 3), strides = (1,1), padding = \"same\", activation='relu'))\n  model.add(MaxPooling2D(pool_size=(3, 3), strides = (2,2)))\n  model.add(Dropout(0.4))\n\n  model.add(Flatten())\n  model.add(Dense(512, activation='relu'))\n\n  model.add(Dropout(0.4))\n  model.add(Dense(512, activation='relu'))\n\n  model.add(Dense(num_classes, activation='softmax'))\n\n\n  return model\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"HLjtFULT6sIE","colab_type":"code","outputId":"285566eb-840a-45b9-d7c0-bbde3b079a1a","colab":{"base_uri":"https://localhost:8080/","height":1037},"trusted":true,"_uuid":"ce59038aa1f873884c88a87bea9551b49e35a3ff"},"cell_type":"code","source":"model_4Conv_3FC = create_model_4Conv_3FC()\n\nadam = optimizers.Adam(lr=1e-4)\n\nmodel_4Conv_3FC.compile(loss='categorical_crossentropy', optimizer= adam, metrics=['accuracy'])\n\n# vanilla model with original training set gave lower validation accuracy\n#model_4Conv_3FC_history = model_4Conv_3FC.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=30, batch_size=128, verbose=2, shuffle=True)          \n\n#with data augmentation  \nmodel_4Conv_3FC_history = model_4Conv_3FC.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=30, validation_data=(X_val, y_val), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"id":"FXyPxFkJ68sW","colab_type":"code","outputId":"83fa42ac-45fb-437c-de9a-553ba9bcee27","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true,"_uuid":"d4a5fe050c4908f7a45d7e904bcd547341f70a08"},"cell_type":"code","source":"#save_files(model_4Conv_3FC, \"model_4Conv_3FC\", model_4Conv_3FC_history)\n\nmodel_evaluate(model_4Conv_3FC_history)","execution_count":null,"outputs":[]},{"metadata":{"id":"xO3oZ8UCMZTc","colab_type":"text","_uuid":"40f0157a6b979013a670b2fb66e057a87906908a"},"cell_type":"markdown","source":"The model with 4 Conv + 3 FC layers gives an appreciable improvement in validation accuracy to 99.36%.\n\nIn training this model, learning rate was lowered to 1e-4, as the default value of 1e-3 resulted in noisy loss curves for which it was difficult to determine whether the training had converged. Dropout values were also tuned and data augmentation was used to reduce overfitting."},{"metadata":{"id":"4-JuCKlbKoaP","colab_type":"text","_uuid":"29f3c723cb5cb5876a283b02e5eaefef51aa84b7"},"cell_type":"markdown","source":"**b) CNN with 6 Conv Layers + 3 FCs** \n\nSimilar to the previous model, a scaled down version of the VGGnet architecture was used. The l VGGnet utilizes only 3x3 filters for conv layers and 2x2 pooling layers. In consideration of the complexity(or lack thereof) of the MNIST data set and the smaller image sizes, fewer convolutional layers were employed for this model."},{"metadata":{"id":"dbfM6uvAKqSD","colab_type":"code","colab":{},"trusted":true,"_uuid":"d8e41e5977969bce03c5213baedda799edde6d5c"},"cell_type":"code","source":"def create_model_6Conv_3FC():\n  seed = 7\n  np.random.seed(seed)\n  \n  model = Sequential()\n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), input_shape=(28,28,1), activation='relu', padding='same'))\n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), activation='relu', padding='same'))\n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), activation='relu', padding='same'))\n  model.add(MaxPooling2D(pool_size=(2, 2), strides = (2,2)))\n  model.add(BatchNormalization())\n  \n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), activation='relu', padding='same'))\n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), activation='relu', padding='same'))\n  model.add(Conv2D(24, kernel_size = (3, 3), strides = (1,1), activation='relu', padding='same'))\n  model.add(MaxPooling2D(pool_size=(2, 2), strides = (2,2)))\n  model.add(BatchNormalization())\n\n  model.add(Flatten())\n  model.add(Dropout(0.2))\n  model.add(Dense(512, activation='relu'))\n  model.add(Dropout(0.2))\n  model.add(Dense(512, activation='relu'))\n  model.add(Dense(num_classes, activation='softmax'))\n\n\n  return model\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"IzEOoza9ZjL0","colab_type":"code","outputId":"bca6ec7b-3720-4d47-85d5-9b831f45a618","colab":{"base_uri":"https://localhost:8080/","height":1037},"trusted":true,"_uuid":"b88ef0e3b38e114209ad064ab08e9d20507ee1d8"},"cell_type":"code","source":"model_6Conv_3FC = create_model_6Conv_3FC()\n\nadam = optimizers.Adam(lr=1e-4)\n\nmodel_6Conv_3FC.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n\n# vanilla model with original data set\n#model_6Conv_3FC_history = model_6Conv_3FC.fit(X_train, y_train, validation_data=(X_val, y_val),\n#                                                  epochs=30, batch_size=128, verbose=2, shuffle=True)\n\n#data augmentation\nmodel_6Conv_3FC_history = model_6Conv_3FC.fit_generator(train_generator, steps_per_epoch=60000//128, epochs=30, validation_data=(X_val, y_val), verbose=2)         ","execution_count":null,"outputs":[]},{"metadata":{"id":"XqwQhL-ibiTA","colab_type":"code","outputId":"291d72c3-29b9-44c9-8c55-b0f68d94319f","colab":{"base_uri":"https://localhost:8080/","height":516},"trusted":true,"_uuid":"251830d6bdf6431c5295b7eff964df0e62186b05"},"cell_type":"code","source":"model_evaluate(model_6Conv_3FC_history)\n#save_files(model_6Conv_3FC, \"model_6Conv_3FC\", model_6Conv_3FC_history)","execution_count":null,"outputs":[]},{"metadata":{"id":"M0l6SZ1TMyU-","colab_type":"text","_uuid":"f3fd529d1a883124210653eb9e22b071a2a55851"},"cell_type":"markdown","source":"The model with 6 Conv + 3 FC layers has a validation accuracy of 99.28%,  comparable to some of rhe earlier models. Similar to the CNN with 4Conv layers, the learning rate was reduced to 1e-4 to produce stabler loss curves to aid in confirmation that the model had been optimized. "},{"metadata":{"id":"D1c9pywiPu1g","colab_type":"code","colab":{},"trusted":true,"_uuid":"62e4ee919e7ad0ad5e6f9b3dec64d1a897fff5ea"},"cell_type":"code","source":"index = [\"lenet_model\",\"lenet_dropout_model\",\"lenet_model_dataAug\",\n          \"model_4Conv_3FC\", \"model_6Conv_3FC\"]\nmodel_history = [lenet_model_history.history,lenet_dropout_model_history.history,\n                 lenet_model_dataAug_history.history,\n                 model_4Conv_3FC_history.history, \n                 model_6Conv_3FC_history.history]\nmodels = [lenet_model,lenet_dropout_model,lenet_model_dataAug,\n          model_4Conv_3FC, model_6Conv_3FC]\n\ntrainingAcc = []\nvalAcc = []\ntrainingLoss = []\nvalLoss = []\nnumParams = []\n\nfor model, hist in zip(models, model_history):\n  trainingAcc.append(round(hist['acc'][-1],4))\n  valAcc.append(round(hist['val_acc'][-1],4))\n  trainingLoss.append(round(hist['loss'][-1],4))\n  valLoss.append(round(hist['val_loss'][-1],4))\n  numParams.append(model.count_params())\n\ndf = pd.DataFrame(list(zip(trainingAcc, valAcc, trainingLoss, valLoss, numParams)), \n               columns =['trainingAcc', 'valAcc', 'trainingLoss', 'valLoss', 'numParams'], \n                 index = index) \n  \n","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"185815ce-c80e-4d53-8f22-ef64dead2a4b","id":"qBA2Tr3S8Lhe","colab":{"base_uri":"https://localhost:8080/","height":204},"trusted":true,"_uuid":"8aed13be49df957ff0b0419f6a77755d140eb51b"},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"id":"gB9_RT0iPTPu","colab_type":"code","colab":{},"trusted":true,"_uuid":"54ac8de3cef9c528cf07883e3c048c475d836922"},"cell_type":"code","source":"predictions = model_4Conv_3FC.predict(X_test_reshaped)\npredictions_classes = np.argmax(predictions, axis=1)\nfilename = \"submission.csv\"\n\npredictions_df = pd.DataFrame({'ImageId': np.arange(1,len(predictions_classes)+1), 'Label': predictions_classes})\npredictions_df.to_csv(filename, index=False)","execution_count":null,"outputs":[]},{"metadata":{"id":"Dx7EFxVcDshl","colab_type":"code","colab":{},"trusted":true,"_uuid":"45d011ed07bd297d4fd88854204b9cc584bd2fcf"},"cell_type":"code","source":"files.download(\"submission.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"id":"N1dW8vOGD21O","colab_type":"code","colab":{},"trusted":false,"_uuid":"92a84663c3ccc40bdb1be348067f6bdeb121b1f0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"MNIST-kaggle.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"accelerator":"GPU","language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}