{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"115412edd64412413dee095221805d05d21f8b88"},"cell_type":"markdown","source":"## Imports.."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7b126374ca5718776f2ce0c5a466b0c089869c5"},"cell_type":"markdown","source":"## Config.."},{"metadata":{"trusted":true,"_uuid":"36c391d160eeee0f1ed21503ea900af28338a6f9"},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"562926ec9b06e3b66cf24e55e063279bae57052d"},"cell_type":"markdown","source":"## Analysis.."},{"metadata":{"trusted":true,"_uuid":"76e7cae0e5f82cca79f6a54a6843128e999a0427"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\n\nlen(train_df), len(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab87f2376470d0c5eb03ce5c8d8da2a43a9e4cfe"},"cell_type":"code","source":"# sample rows\n\nprint(train_df.head())\nprint(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d54c348db47e987ab1e8e760d2e02f489884e6f2"},"cell_type":"markdown","source":"**Observations**\n1. Training data consists of \"label\" and 784 pixel values as column names ranging from \"pixel0\" to \"pixel783\"\n2. Test has just 784 columns with all pixel values without labels"},{"metadata":{"trusted":true,"_uuid":"0305cfffdb9fbf2e2a2df02a3f8920455e15f230"},"cell_type":"code","source":"28*28","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0305cfffdb9fbf2e2a2df02a3f8920455e15f230"},"cell_type":"markdown","source":"**Observations**\n1. Each row is an image with 28 pixels width and 28 pixels height. Each pixel ranges from (0, 255) representing grayscale"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"34bd611caa41898fbfed4394a148c332aaa7547f"},"cell_type":"code","source":"# sample train images\n\nnp.random.seed(13)\nfig, axes = plt.subplots(nrows=2, ncols=4, figsize=(20, 10))\nrandom_rows = np.random.choice(train_df.index, size=8, replace=False)\nfor ax, idx in zip(axes.flat, random_rows):\n    row = train_df.iloc[idx, 1:].values\n    img_matrix = row.reshape(28, 28)\n    ax.imshow(img_matrix, cmap=\"gray\")\n    ax.set_title(train_df.iloc[idx, 0], fontdict={\"fontsize\": 24})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10a47d04e51099a92569f5854003095ae22f62d8"},"cell_type":"markdown","source":"**Observations**\n1. **2** at position (2, 2) and **4** at (1, 1) are difficult for a human to judge as well"},{"metadata":{"trusted":true,"_uuid":"bfe4527b3fa51ad388884cfb24410941ddd849d3","scrolled":true},"cell_type":"code","source":"# sample test images\n\nfig, axes = plt.subplots(nrows=1, ncols=4, figsize=(20, 5))\nnp.random.seed(13)\nrandom_rows = np.random.choice(test_df.index, size=4, replace=False)\nfor ax, idx in zip(axes.flat, random_rows):\n    row = test_df.iloc[idx, :]\n    ax.imshow(row.values.reshape(28, 28), cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3537f124e610651f2948afe28a3268b25c72e92"},"cell_type":"markdown","source":"**Observations**\n1. Any guess what is at (1, 4)... 8 probably??"},{"metadata":{"trusted":true,"_uuid":"be37cef0a5967a20c438e30e1766691837559dd2"},"cell_type":"markdown","source":"## Model building"},{"metadata":{"trusted":true,"_uuid":"b350c79c9719b7fd79ec69ac98d3f7442942f04e"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\nfrom keras.layers import Conv2D, Dense, Flatten, MaxPool2D, Activation\nfrom keras.layers import Dropout, BatchNormalization\nfrom keras.optimizers import Adam, RMSprop\nfrom keras.models import Sequential\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18f818ede38b4dd4420d750ed7a7be9aa19e296b"},"cell_type":"markdown","source":"**Observations**\n1. Rescaling inputs is essential especially when using RELU's."},{"metadata":{"trusted":true,"_uuid":"efe438e56950d92d59d3bf266c579917ebeab520"},"cell_type":"code","source":"# rescale inputs\ntrain_df.iloc[:, 1:] /= 255\ntest_df /= 255\n\ntrain_inputs = train_df.iloc[:, 1:].values.reshape(len(train_df), 28, 28, 1)\ntest_inputs = test_df.iloc[:, :].values.reshape(len(test_df), 28, 28, 1)\n\ntrain_inputs.shape, test_inputs.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dacf16c14c8a3029f1bad475b074cea23743b34e"},"cell_type":"markdown","source":"One hot encode the \"label\" column to represent each value with a vector of size 10"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"aa01e1ee9c1839a57909dd4973647af605189995"},"cell_type":"code","source":"one_hot_labels = to_categorical(train_df.label.values, num_classes=10)\none_hot_labels[: 5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be916345ccfe443a2d1954701360d74d78c99e70"},"cell_type":"code","source":"# Split train - validation\nX_train, X_val, y_train, y_val = train_test_split(train_inputs, one_hot_labels, test_size=0.2, random_state=13)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"487a8a8e909845d4c9d1dac1d591112d39be3679"},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), input_shape=(28, 28, 1), name=\"conv1\", padding=\"same\"))\nmodel.add(Activation(\"relu\", name=\"act1\"))\nmodel.add(Conv2D(32, (3, 3), input_shape=(28, 28, 1), name=\"conv2\", padding=\"same\"))\nmodel.add(Activation(\"relu\", name=\"act2\"))\nmodel.add(MaxPool2D((2, 2), name=\"mpool1\"))\nmodel.add(Dropout(0.25, name=\"drop1\"))\n\nmodel.add(Conv2D(64, (3, 3), name=\"conv3\", padding=\"same\"))\nmodel.add(Activation(\"relu\", name=\"act3\"))\nmodel.add(Conv2D(64, (3, 3), name=\"conv4\", padding=\"same\"))\nmodel.add(Activation(\"relu\", name=\"act4\"))\nmodel.add(MaxPool2D((2, 2), name=\"mpool2\"))\nmodel.add(Dropout(0.25, name=\"drop2\"))\n\nmodel.add(Flatten(name=\"flat1\"))\nmodel.add(BatchNormalization(name=\"bn1\"))\nmodel.add(Dense(512, activation=\"relu\", name=\"dense1\"))\nmodel.add(Dropout(0.5, name=\"drop3\"))\n\nmodel.add(Dense(10, activation=\"softmax\", name=\"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"38252dd42fbd8463e76a6cf8288a1eb9a9d1475f"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"619aa3f7ad7eb0778a852bdeaeacf00c2945758e"},"cell_type":"code","source":"# Using Keras 'ImageDataGenerator'. Function returns a generator with images transformed\ndatagen = ImageDataGenerator(zoom_range = 0.1,\n                             height_shift_range = 0.1,\n                             width_shift_range = 0.1,\n                             rotation_range = 10)\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"619aa3f7ad7eb0778a852bdeaeacf00c2945758e"},"cell_type":"code","source":"# By using Keras callbacks, we can control the learning rate automatically while model is getting built\n# Below ReduceLROnPlateau decreases learning rate based on change in validation accuracy\n# patience - waits for 3 epochs\n# min_lr - if learning rate is not changed by \"min_lr\" amount\n# factor - existing learning rate by this value\n\nlr_reduce = ReduceLROnPlateau(monitor='val_acc', patience=3, factor=0.5, verbose=1, min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d257a4144b69a17be1616cf1bdb7b85ddd07378b"},"cell_type":"code","source":"model.compile(optimizer=Adam(), metrics=[\"accuracy\"], loss=\"categorical_crossentropy\")\nmodel.fit_generator(datagen.flow(X_train, y_train, batch_size=64),\n                    validation_data=(X_val, y_val), epochs=100, callbacks=[lr_reduce], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdafafce6f841748f0ac4c78f4dcc92d3828a162"},"cell_type":"code","source":"# predictions\ntest_preds = model.predict(test_inputs)\ntest_labels = np.argmax(test_preds, axis=1)\ntest_labels[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"2a7bd9986ad9f0478ee42541f91c99a256a4c672"},"cell_type":"code","source":"# submission dataframe\nsub_df = pd.concat([pd.Series(range(1, 28001), name=\"ImageId\"), pd.Series(test_labels, name=\"Label\")], axis=1)\n\n# save submission\nsub_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}