{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom keras import layers\nfrom keras.layers import Input, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D\nfrom keras.layers import AveragePooling2D, MaxPooling2D, Dropout, GlobalMaxPooling2D, GlobalAveragePooling2D\nfrom keras.models import Model\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3856432094d1452508b23fc411c72228d86cb8a"},"cell_type":"markdown","source":"## Load the datasets and split into X and Y"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Load the train and test datasets\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25ff15104ffce8c230deb846f275804049b7a0a2"},"cell_type":"code","source":"# Train dataset has 'label' in the first column\n# Split it and make X and Y\nY_train_orig = train['label']\nX_train_orig = train.drop(labels=['label'], axis=1)\nprint(f'Shape of X is: {X_train_orig.shape}')\nprint(f'Shape of Y is: {Y_train_orig.shape}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39a9122ed629fe8eebd68abd867ba5fc0680153c"},"cell_type":"markdown","source":"## Standardization, Reshaping for Keras, and One-hot encoding\n\n- Standardize the pixel values so that all the X values are between 0 and 1 instead of between 0 and 255\n- Convert Y values into 1-hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\n- Split the training set into Train and Dev datasets. Reshape dataset into 4 dimentional matrix where\n    *     m: number of training examples\n    *     n_h: pixels represnting height of image\n    *     n_w: pixels representing width of image\n    *     n_c: pixels representing the RGB channels"},{"metadata":{"trusted":true,"_uuid":"0c2f00cc8b00a4a0a3e4f1177c312cf13002e546"},"cell_type":"code","source":"# Standardize dataset. Divide by 255 to get all x values between 0 and 1\nX_train = X_train_orig / 255.\ntest = test / 255.\n\n# Convert Y to one-hot vectors\nY_train = pd.get_dummies(Y_train_orig)\n\n# Reshape X to dimensions (m, n_h, n_w, n_c)\nX_train = X_train.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1, 28, 28, 1)\n\nprint(X_train.shape)\nprint(Y_train.shape)\n\nX_train, X_dev, Y_train, Y_dev = train_test_split(X_train, Y_train, test_size=0.1, random_state=42)\n\n# X_train, X_dev, Y_train, Y_dev = X_train.T, X_dev.T, Y_train.T, Y_dev.T\nprint(X_train.shape)\nprint(Y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87b0d9d81ba69b010e4d81452eba00231230d82d"},"cell_type":"markdown","source":"## View a sample image and the label that is assigned to it"},{"metadata":{"trusted":true,"_uuid":"ced6c9d1147d8beac4769dd820c3805965669b52"},"cell_type":"code","source":"index = 560\nplt.figure()\nplt.imshow(X_train[index][:,:,0])\nplt.colorbar()\nplt.grid(False)\nprint (\"y = \" + str(np.squeeze(Y_train.values[index, :])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a3eca2d30cfbfa8ae5be6d1d639e62718f1055a"},"cell_type":"code","source":"def model(input_shape):\n    # Define the input placeholder as a tensor with shape input_shape. Think of this as your input image!\n    X_input = Input(input_shape)\n\n    # Zero-Padding: pads the border of X_input with zeroes\n    X = ZeroPadding2D((3, 3))(X_input)\n\n    # CONV -> BN -> RELU Block applied to X\n    X = Conv2D(32, (7, 7), strides = (1, 1), name = 'conv0')(X)\n    X = BatchNormalization(axis = 3, name = 'bn0')(X)\n    X = Activation('relu')(X)\n\n    # MAXPOOL\n    X = MaxPooling2D((2, 2), name='max_pool0')(X)\n    \n    # CONV -> BN -> RELU Block applied to X\n    X = Conv2D(64, (7, 7), strides = (1, 1), name = 'conv1')(X)\n    X = BatchNormalization(axis = 3, name = 'bn1')(X)\n    X = Activation('relu')(X)\n\n    # MAXPOOL\n    X = MaxPooling2D((2, 2), name='max_pool1')(X)\n\n    # FLATTEN X (means convert it to a vector) + FULLYCONNECTED\n    X = Flatten()(X)\n    X = Dense(128, activation='relu', name='fc0')(X)\n    X = Dense(128, activation='relu', name='fc1')(X)\n    X = Dense(10, activation='softmax', name='fc2')(X)\n\n    # Create model. This creates your Keras model instance, you'll use this instance to train/test the model.\n    model = Model(inputs = X_input, outputs = X, name='DigitRecognizer')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99edfc16e04fee792fc8d461adbaa8f4f8bf4523"},"cell_type":"markdown","source":"## Data Augmentation\n\nIn order to avoid overfitting the model to the training set, we need to add more training data. We do this by generating synthetic data from the provided dataset by -\n1. Rotating the images by a few degrees  \n2. Zooming into the images by a few factors\n3. Shifting images vertically or horizontally"},{"metadata":{"trusted":true,"_uuid":"db350d23bb7807842678f8eeca86b5d90bfb5929"},"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range=10,\n    zoom_range=0.1,\n    width_shift_range=0.1,\n    height_shift_range=0.1, \n    horizontal_flip=False,\n    vertical_flip=False)\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88f651cca50b12db4cf6b29a6dc89ef03f772f9e"},"cell_type":"markdown","source":"## Annealing Method\n\nManage the learning rate such that if there is no significant improvement in the accuracy over few epoch, reduce the learning rate so that the steps taken by the model are smaller and helps in getting to the minima faster instead of oscillating around it."},{"metadata":{"trusted":true,"_uuid":"04ad9a012188766084b33db6d008d31cb9df0fc7"},"cell_type":"code","source":"# Trying some things for improving performance\nreduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=3, min_lr=0.00001, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efb473cd91c2b1b0fc273862c6f85a6cf7752a5d"},"cell_type":"markdown","source":"## Train model and validate on dev dataset to see generalization performance."},{"metadata":{"trusted":true,"_uuid":"8c4e4ce035ab34802370f248edd396e331377b64"},"cell_type":"code","source":"print(X_train.shape[1:])\ndigit_recognizer = model(X_train.shape[1:])\ndigit_recognizer.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])\n# digit_recognizer.fit(x = X_train, y = Y_train, epochs = 5, batch_size = 16, validation_data=(X_dev, Y_dev))\ndigit_recognizer.fit_generator(datagen.flow(X_train, Y_train, batch_size=32), \n                               epochs=30, \n                               steps_per_epoch=len(X_train) / 32, \n                               validation_data=(X_dev, Y_dev), \n                               callbacks=[reduce_lr])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c846e0c88447ca57109cb047a9ca0b508516d458"},"cell_type":"markdown","source":"## Generate output predictions for the test samples"},{"metadata":{"trusted":true,"_uuid":"f6a039db65d81bdf1578e626821779c3bc1b2026"},"cell_type":"code","source":"# predict results\nresults = digit_recognizer.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9532246dea340c5b8609b90dcaa08b9816407aa4"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"digit_recognizer_cnn.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69a107d0673518cfbf93c0a8ee9e779dd4e68443"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}