{"cells":[{"metadata":{"_cell_guid":"84d4608d-4cc3-fcbb-57fb-61f07ad7d020","_uuid":"6407080d145a62b4803d7f159c00118a056a7b5f"},"cell_type":"markdown","source":"*THanks Poonam Ligade*\n\n*1st Feb 2017*\n\n\n----------\n\n\nThis notebook is like note to self.\n\nI am trying to understand various components of Artificial Neural Networks aka Deep Learning.\n\nHope it might be useful for someone else here.\n\nI am designing neural net on MNIST handwritten digits images to identify their correct label i.e number in image.\n\nYou must have guessed its an image recognition task.\n\nMNIST is called Hello world of Deep learning.\n\nLets start!!\n\nThis notebook is inspired from [Jeremy's][1] [Deep Learning][2] mooc and [Deep learning with python][3] book by Keras author [François Chollet][4] .\n\n\n  [1]: https://www.linkedin.com/in/howardjeremy/\n  [2]: http://course.fast.ai/\n  [3]: https://www.manning.com/books/deep-learning-with-python\n  [4]: https://research.google.com/pubs/105096.html"},{"metadata":{"_cell_guid":"654456b6-e648-0379-0d66-1cc97af6d00d","_uuid":"6b48ce0e361bdb67689dd2f254ecedd9ade1f5ff"},"cell_type":"markdown","source":"**Import all required libraries**\n==============================="},{"metadata":{"_cell_guid":"e5b02688-c589-5a89-e11c-837c6a99eb6e","_uuid":"f043e48097bfd98e41710142dd8aac41fa88a801","trusted":true},"cell_type":"code","source":"    # This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense , Dropout , Lambda, Flatten\nfrom keras.optimizers import Adam ,RMSprop\nfrom sklearn.model_selection import train_test_split\nfrom keras import  backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"22a7fd70-ab61-432d-24cb-93e558414495","_uuid":"62fbd0fe9c338b7ac0b04e688c8ee7947e6170f7"},"cell_type":"markdown","source":"**Load Train and Test data**\n============================"},{"metadata":{"_cell_guid":"05226b08-226a-1a00-044d-a0e6b2101388","_uuid":"4eff577bcd43479a3b7e91180393cbad9fcfca33","trusted":true},"cell_type":"code","source":"# create the training & test sets, skipping the header row with [1:]\ntrain = pd.read_csv(\"../input/train.csv\")\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2ec570a6-b41a-2139-5e0e-4941c4f0a9d0","_uuid":"67f0854ad0d812a1395130144a0adef9966fec88","trusted":true},"cell_type":"code","source":"test= pd.read_csv(\"../input/test.csv\")\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1ae10fe0-dde9-7659-f53d-1a1bd625cfb1","_uuid":"bdffbed77ce62da528c60e43f2b1bea9f57fcdbc","trusted":true},"cell_type":"code","source":"X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\ny_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\nX_test = test.values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"250b1126-ce1d-6d3f-9736-2504f7a1e098","_uuid":"5e3e1e3574c3e019eadfd14e4dda41fd15b4de2a","trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e0f15f8a-ac08-540a-58db-dab989cc687c","_uuid":"4c96cf1c9cdc364ae3faff6b8c3c97aa7fa982d4","trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c2c91588-5547-353a-7f92-39600027438e","_uuid":"f01a969286e62fa5ffe37031ed6d4aea947b59a8"},"cell_type":"markdown","source":"The output variable is an integer from 0 to 9. This is a **multiclass** classification problem."},{"metadata":{"_cell_guid":"60957d82-c76f-4822-28ff-def7011a34fa","_uuid":"da0573528e8c6c3dd2b0e0cf33c600ad3f14466d"},"cell_type":"markdown","source":"## Data Visualization\nLets look at 3 images from data set with their labels."},{"metadata":{"_cell_guid":"1541678d-a08b-d2b2-1e1e-eabf882baaec","_uuid":"7998af1ce3c065c4a54a73cd97fed9afebde7e96","trusted":true},"cell_type":"code","source":"#Convert train datset to (num_images, img_rows, img_cols) format \nX_train = X_train.reshape(X_train.shape[0], 28, 28)\n\nfor i in range(6, 9):\n    plt.subplot(330 + (i+1))\n    plt.imshow(X_train[i], cmap=plt.get_cmap('gray'))\n    plt.title(y_train[i]);","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6be2f3e9-42eb-85b6-9162-c25e4d706155","_uuid":"4051a0e6612b8e6d4b8aef6a4d131be621cd3a14","trusted":true},"cell_type":"code","source":"#expand 1 more dimention as 1 for colour channel gray\nX_train = X_train.reshape(X_train.shape[0], 28, 28,1)\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6949468c-fd27-19c5-15c7-0b357a961003","_uuid":"6d4c1323f1fa3f89a16532fed893ec5d72051bcb","trusted":true},"cell_type":"code","source":"X_test = X_test.reshape(X_test.shape[0], 28, 28,1)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1232c385-3cb2-56fd-4d1d-f027df7bc78e","_uuid":"185d620525e041eb61aabce19e8536614ab50870"},"cell_type":"markdown","source":"**Preprocessing the digit images**\n=================================="},{"metadata":{"_cell_guid":"6fcc1f9e-1586-e393-49ba-50c73564e0ed","_uuid":"b8847f48f7408c93ce795db16f30c1b7c6a8cf89"},"cell_type":"markdown","source":"**Feature Standardization**\n-------------------------------------\n\nIt is important preprocessing step.\nIt is used to centre the data around zero mean and unit variance."},{"metadata":{"_cell_guid":"a3f837ef-0373-8d91-46e6-30992cf73166","_uuid":"528a370b381c91b73131a8c7a4217968278696c8","trusted":true},"cell_type":"code","source":"mean_px = X_train.mean().astype(np.float32)\nstd_px = X_train.std().astype(np.float32)\n\ndef standardize(x): \n    return (x-mean_px)/std_px","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"725c55fc-9742-a63c-9822-c67ab0c773ee","_uuid":"532d3f3bd26b0dfb42bc0c96e9710269234fae9b"},"cell_type":"markdown","source":"*One Hot encoding of labels.*\n-----------------------------\n\nA one-hot vector is a vector which is 0 in most dimensions, and 1 in a single dimension. In this case, the nth digit will be represented as a vector which is 1 in the nth dimension. \n\nFor example, 3 would be [0,0,0,1,0,0,0,0,0,0]."},{"metadata":{"_cell_guid":"c879f076-b3dd-6cb1-e2d9-2f404f2ed132","_uuid":"41bb3082e71111d73dd0432f9f60261f5be05e15","trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\ny_train= to_categorical(y_train)\nnum_classes = y_train.shape[1]\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4d76fb04-57fc-e802-6d91-06ece552686b","_uuid":"429e528f5bf36152cd9e0b2acaa457525a202171"},"cell_type":"markdown","source":"Lets plot 10th label."},{"metadata":{"_cell_guid":"1c927e75-08d2-d539-54f3-71ab0308fec1","_uuid":"b3ad8362611417de16de730cc55a6ff6309766f4","trusted":true},"cell_type":"code","source":"plt.title(y_train[9])\nplt.plot(y_train[9])\nplt.xticks(range(10));","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4e130661-9f09-d9a9-d49b-7274ef13927f","_uuid":"40aecbff9b92269d438c384ac429bfa47ab37dda"},"cell_type":"markdown","source":"Oh its 3 !"},{"metadata":{"_cell_guid":"6a89dcdd-7b68-6ed1-2c39-b3a1edb3e7be","_uuid":"dc7ece2b7ee08767b664149d67d922d8c1d0bbb1"},"cell_type":"markdown","source":"**Designing Neural Network Architecture**\n========================================="},{"metadata":{"_cell_guid":"39107235-d87a-af4d-44fb-80c9c3aa0212","_uuid":"1070353d05490ccec23933c62f11cdfd2d7e5032","trusted":true},"cell_type":"code","source":"# fix random seed for reproducibility\nseed = 43\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a8b65f54-398b-267f-e31a-313210450f54","_uuid":"62606ecbb1d7e259850aebf8a8514e54263a2a06"},"cell_type":"markdown","source":"*Linear Model*\n--------------"},{"metadata":{"_cell_guid":"5dbe450c-845f-aaa2-dbde-21414a91d8c1","_uuid":"5f54b59d89cd4e43dd129d9b133950ba83b5cad8","trusted":true},"cell_type":"code","source":"from keras.models import  Sequential\nfrom keras.layers.core import  Lambda , Dense, Flatten, Dropout\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import BatchNormalization, Convolution2D , MaxPooling2D","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5c3f674f-f3fc-9614-f2d4-056c3e3ad633","_uuid":"ff25a88562237e84e44f20b38079f2b44a394d2c"},"cell_type":"markdown","source":"Lets create a simple model from Keras Sequential layer.\n\n1. Lambda layer performs simple arithmetic operations like sum, average, exponentiation etc.\n\n In 1st layer of the model we have to define input dimensions of our data in (rows,columns,colour channel) format.\n (In theano colour channel comes first)\n\n\n2. Flatten will transform input into 1D array.\n\n\n3. Dense is fully connected layer that means all neurons in previous layers will be connected to all neurons in fully connected layer.\n In the last layer we have to specify output dimensions/classes of the model.\n Here it's 10, since we have to output 10 different digit labels."},{"metadata":{"_cell_guid":"a2c27783-3cfa-e907-4749-1e340a513f26","_uuid":"fb79b4558335446a722542c8bc06288e96781423","trusted":true},"cell_type":"code","source":"model= Sequential()\nmodel.add(Lambda(standardize,input_shape=(28,28,1)))\nmodel.add(Flatten())\nmodel.add(Dense(10, activation='softmax'))\nprint(\"input shape \",model.input_shape)\nprint(\"output shape \",model.output_shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"260645fb-61b7-68e9-6826-047b97436c14","_uuid":"2dd7f371688dd2590de94a94814799654595e55d"},"cell_type":"markdown","source":"***Compile network***\n-------------------\n\nBefore making network ready for training we have to make sure to add below things:\n\n 1.  A loss function: to measure how good the network is\n    \n 2.  An optimizer: to update network as it sees more data and reduce loss\n    value\n    \n 3.  Metrics: to monitor performance of network"},{"metadata":{"_cell_guid":"9d1d1af9-b2a8-e3b9-6eaf-100d08fe83aa","_uuid":"4bb75be10b9eec8bcdfe48665f639bf326a5f5fc","trusted":true},"cell_type":"code","source":"from keras.optimizers import RMSprop\nmodel.compile(optimizer=RMSprop(lr=0.001),\n loss='categorical_crossentropy',\n metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"db3b4be6-4f72-c6cc-65cd-b45978db2462","_uuid":"51f82558d87e95fa5c146b0469ab6c8b42e13bcf","trusted":true},"cell_type":"code","source":"from keras.preprocessing import image\ngen = image.ImageDataGenerator()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"841a6f3b78b607e142f3e18d88bd7957202e4dcb"},"cell_type":"markdown","source":"## Cross Validation "},{"metadata":{"_cell_guid":"9071d720-da50-8530-e9f3-1f0c37aac7ff","_uuid":"0cff7e02b1ee8894b4ee9080b9268558aaa4e7c5","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX = X_train\ny = y_train\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\nbatches = gen.flow(X_train, y_train, batch_size=64)\nval_batches=gen.flow(X_val, y_val, batch_size=64)\nprint(batches.n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"20e08e2a-a394-bb70-69f1-be0fdab4f9ab","_uuid":"6b23c282e2772b1ee482596131d6f1d3494c3bce","trusted":true},"cell_type":"code","source":"history=model.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=3, \n                    validation_data=val_batches, validation_steps=val_batches.n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9f344366-c372-0b04-b7e0-860778d4bfd3","_uuid":"6900e38c62028692b9f101b94730c527129675cc","trusted":true},"cell_type":"code","source":"history_dict = history.history\nhistory_dict.keys()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"df40f5fc-586a-1fae-025e-ee508a8d9b71","_uuid":"c4b26ff79e0f186212266b60d03611ad58d0d5e3","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nloss_values = history_dict['loss']\nval_loss_values = history_dict['val_loss']\nepochs = range(1, len(loss_values) + 1)\n\n# \"bo\" is for \"blue dot\"\nplt.plot(epochs, loss_values, 'bo')\n# b+ is for \"blue crosses\"\nplt.plot(epochs, val_loss_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1ed6b756-00c2-d08c-c596-0ce496ec3d04","_uuid":"fc9be5b885360ca9972e0d6b5da1ea36dc12cb5f","trusted":true},"cell_type":"code","source":"plt.clf()   # clear figure\nacc_values = history_dict['acc']\nval_acc_values = history_dict['val_acc']\n\nplt.plot(epochs, acc_values, 'bo')\nplt.plot(epochs, val_acc_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64ec304e056ec0c9e33fe94ea2315cbf65a7fbff"},"cell_type":"markdown","source":"## Fully Connected Model\n\nNeurons in a fully connected layer have full connections to all activations in the previous layer, as seen in regular Neural Networks. \nAdding another Dense Layer to model."},{"metadata":{"trusted":true,"_uuid":"9556f3de5bd370bcddc70a81910eb2104624e3a3"},"cell_type":"code","source":"def get_fc_model():\n    model = Sequential([\n        Lambda(standardize, input_shape=(28,28,1)),\n        Flatten(),\n        Dense(512, activation='relu'),\n        Dense(10, activation='softmax')\n        ])\n    model.compile(optimizer='Adam', loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1901b6805f878ac6ed4efafbaf15bf003d505654"},"cell_type":"code","source":"fc = get_fc_model()\nfc.optimizer.lr=0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fb346c542c8920fac61ddc5df44b2136969a6e9"},"cell_type":"code","source":"history=fc.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=1, \n                    validation_data=val_batches, validation_steps=val_batches.n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46b81b17854a98f2b380da694691502c1e583bfb"},"cell_type":"markdown","source":"## Convolutional Neural Network\nCNNs are extremely efficient for images.\n"},{"metadata":{"trusted":true,"_uuid":"fd0ab0de6bdbf7addf7515c8d7b59d8d17fef8e7"},"cell_type":"code","source":"from keras.layers import Convolution2D, MaxPooling2D\n\ndef get_cnn_model():\n    model = Sequential([\n        Lambda(standardize, input_shape=(28,28,1)),\n        Convolution2D(32,(3,3), activation='relu'),\n        Convolution2D(32,(3,3), activation='relu'),\n        MaxPooling2D(),\n        Convolution2D(64,(3,3), activation='relu'),\n        Convolution2D(64,(3,3), activation='relu'),\n        MaxPooling2D(),\n        Flatten(),\n        Dense(512, activation='relu'),\n        Dense(10, activation='softmax')\n        ])\n    model.compile(Adam(), loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5baeaed50c9f03c68900461555f4b7831a7e7c7"},"cell_type":"code","source":"model= get_cnn_model()\nmodel.optimizer.lr=0.01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3d15eeead8b26c41b53d084dc111ae9e667a169"},"cell_type":"code","source":"history=model.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=1, \n                    validation_data=val_batches, validation_steps=val_batches.n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2891c7e434a2022ee182a0e9bd243a876532dcc"},"cell_type":"markdown","source":"## Data Augmentation\nIt is tehnique of showing slighly different or new images to neural network to avoid overfitting. And  to achieve better generalization.\nIn case you have very small dataset, you can use different kinds of data augmentation techniques to increase your data size. Neural networks perform better if you provide them more data.\n\nDifferent data aumentation techniques are as follows:\n1. Cropping\n2. Rotating\n3. Scaling\n4. Translating\n5. Flipping \n6. Adding Gaussian noise to input images etc.\n"},{"metadata":{"trusted":true,"_uuid":"daa409b92678202cf7c751371b7ba17fb14aa2ac"},"cell_type":"code","source":"gen =ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n                               height_shift_range=0.08, zoom_range=0.08)\nbatches = gen.flow(X_train, y_train, batch_size=64)\nval_batches = gen.flow(X_val, y_val, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f21ba7b8d77a37bee6e8238a8f517b654ae3f0a0"},"cell_type":"code","source":"model.optimizer.lr=0.001\nhistory=model.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=1, \n                    validation_data=val_batches, validation_steps=val_batches.n/64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"538f504c44e14d389c70b2f35b7225de61b9015d"},"cell_type":"markdown","source":"## Adding Batch Normalization\n\nBN helps to fine tune hyperparameters more better and train really deep neural networks."},{"metadata":{"trusted":true,"_uuid":"8b72580fbb06f5f4f769c514cb0d7d2f15aa2c2f"},"cell_type":"code","source":"from keras.layers.normalization import BatchNormalization\n\ndef get_bn_model():\n    model = Sequential([\n        Lambda(standardize, input_shape=(28,28,1)),\n        Convolution2D(32,(3,3), activation='relu'),\n        BatchNormalization(axis=1),\n        Convolution2D(32,(3,3), activation='relu'),\n        MaxPooling2D(),\n        BatchNormalization(axis=1),\n        Convolution2D(64,(3,3), activation='relu'),\n        BatchNormalization(axis=1),\n        Convolution2D(64,(3,3), activation='relu'),\n        MaxPooling2D(),\n        Flatten(),\n        BatchNormalization(),\n        Dense(512, activation='relu'),\n        BatchNormalization(),\n        Dense(10, activation='softmax')\n        ])\n    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78e382d0b3de14312e762edc480b5d215be82269"},"cell_type":"code","source":"model= get_bn_model()\nmodel.optimizer.lr=0.01\nhistory=model.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=1, \n                    validation_data=val_batches, validation_steps=val_batches.n/64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8e4b16516a57e152a911f6e7ba7f4d70ff204512"},"cell_type":"markdown","source":"## Submitting Predictions to Kaggle.\nMake sure you use full train dataset here to train model and predict on test set.\n"},{"metadata":{"trusted":true,"_uuid":"0fc055b482971b36561aaf9421c8a9c53df2900b"},"cell_type":"code","source":"model.optimizer.lr=0.01\ngen = image.ImageDataGenerator()\nbatches = gen.flow(X, y, batch_size=64)\nhistory=model.fit_generator(generator=batches, steps_per_epoch=batches.n/64, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c2841d54-f3dd-1ee8-a30d-4457dec0a67a","_uuid":"4262c6bfb15ec96993e83bd2a2552eadf14fb33d","trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(X_test, verbose=0)\n\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                         \"Label\": predictions})\nsubmissions.to_csv(\"DR.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0e6213b0-fc56-658d-46e3-4a5dcb7148ce","_uuid":"3a9a548a2080ebf61b2ce35db78f0c9520c1358c"},"cell_type":"markdown","source":"More to come . Please upvote if you find it useful.\n\nYou can increase number of epochs on your GPU enabled machine to get better results."}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}