{"cells":[{"metadata":{"_uuid":"f67e22e78adb9db658a76b65ec5019cdf7d8a563"},"cell_type":"markdown","source":"# CNN Filter Visualisation\n### In this notebook we will use the MNIST dataset and keras to construct a deep learning model based on CNNs and then try to visualise the outputs of all the filters in the model. This will help us to understand how the image is intercepted at each level of our model."},{"metadata":{"_uuid":"8250c90c54f06b3f56e5b61e6af475cdc3ffc858"},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\nfrom keras import models\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop ,Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nsns.set(style='white', context='notebook', palette='deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"999895531c5b1686919e515519c2d71668401c5e"},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be8c2da19202112222a70b155c4fba4c5a8cd32b"},"cell_type":"code","source":"print(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97148358df2918eaaf0dbbfe931253a6ae1ab126"},"cell_type":"code","source":"print(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc8dd964df5c34a6cf5ea1493cbaeae4c1e78fe1"},"cell_type":"markdown","source":"## Visualise Data Distribution"},{"metadata":{"trusted":true,"_uuid":"aa0bb1e46db200b629b0ab48269a704e47fa015b"},"cell_type":"code","source":"Y_train = train[\"label\"]\n\n# Drop the 'label' column from training set\nX_train = train.drop(labels = [\"label\"],axis = 1) \n\ng = sns.countplot(Y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37e22254d2adce85ec029f1389c97420de390706"},"cell_type":"markdown","source":"## Normalize the Data"},{"metadata":{"trusted":true,"_uuid":"bca4e6c7756687b257f8dc34a3fc30017eba3e18"},"cell_type":"code","source":"# Normalize the data\nX_train = X_train / 255.0\ntest = test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d8931d2a497011491849524d4a1e8b7871495c8"},"cell_type":"markdown","source":"## Reshaping the input to shape 28*28*1"},{"metadata":{"trusted":true,"_uuid":"f4288ea3dd487ae33cc5dd1e9c0c668842123578"},"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nX_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52b43aa4cb050689f21ce2e316e62bfbb34848e5"},"cell_type":"markdown","source":"## Convert the true labels to one hot vectors"},{"metadata":{"trusted":true,"_uuid":"2286a1fdb1ff1dc421c4a3a6f1f6807304bf313c"},"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\nY_train = to_categorical(Y_train, num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bfee3f6117a35a6a4d87ad5ce4c00063a621249"},"cell_type":"markdown","source":"## Train-Val Split"},{"metadata":{"trusted":true,"_uuid":"07c0dcc380c7aa973e574063a6b0655a70683424"},"cell_type":"code","source":"# Set the random seed\nrandom_seed = 2\n# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc7b0cb8362581994c25298b8b0077126546d314"},"cell_type":"code","source":"print(X_train.shape)\nprint(Y_train.shape)\nprint(X_val.shape)\nprint(Y_val.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1247a037c949f9f5807376d0713dbf43a01c9197"},"cell_type":"markdown","source":"### Some examples"},{"metadata":{"trusted":true,"_uuid":"6d5f1485e15a02ae85977d428e36eda9f6fb3d8c"},"cell_type":"code","source":"#g = plt.imshow(X_train[10][:,:,0])\nrow = 5\ncol = 5\nf, axarr = plt.subplots(row,col)\nfor i in range(row):\n    for j in range(col):\n        axarr[i,j].imshow(X_train[i+j][:,:,0]) \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a086ac5394de317d59285f7d51dec02ff58ac2b7"},"cell_type":"markdown","source":"## Define the CNN Model"},{"metadata":{"trusted":true,"_uuid":"9d5358378d2d48e9983903e85a53e4f8310bacc9"},"cell_type":"code","source":"# Set the CNN model \n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5a572a27455fefc069e7b39eff30320bc14bd76"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f685dcfb175deae43b048919559dbc344d7407dd"},"cell_type":"markdown","source":"## Set model parameters"},{"metadata":{"trusted":true,"_uuid":"ae1f70e5fc7eedf3240b3a692580053de211c292"},"cell_type":"code","source":"# Define the optimizer\noptimizer = Adam()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"619809c9a5cc750a13b8d9cc316ba79bfe292972"},"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bba63b0a6de8421b1e5df973057ffa116e3def2"},"cell_type":"code","source":"# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a497ae25e51f595e1b760acefb78e6691aa59e8e"},"cell_type":"code","source":"epochs = 35 # Turn epochs to 30 to get 0.9967 accuracy\nbatch_size = 64","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e89290606ded0fd90b390b0468dc6b1efb8e0ad8"},"cell_type":"markdown","source":"## Define the data generator including augmentation"},{"metadata":{"trusted":true,"_uuid":"210a1bd7279880a2d0adc518f12da780c1fed01a"},"cell_type":"code","source":"# With data augmentation to prevent overfitting (accuracy 0.99286)\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a3849105bd5f9ed1c0e2950f7b6ea7bc5372584"},"cell_type":"markdown","source":"## Training the model"},{"metadata":{"trusted":true,"_uuid":"418eb6d171a43a3e66ed5684c2f0d8645de3bdf4"},"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f690ed8935e995c2b8a8049a55f1c7db8a4d947e"},"cell_type":"markdown","source":"## Plot the training and validation loss and accuracy"},{"metadata":{"trusted":true,"_uuid":"ffc257a161c2945924281834e83a7a842cf7f0e1"},"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nfig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcf5fdb4874c12cdffa8060a7031c7ba89cd0a4c"},"cell_type":"markdown","source":"## Plot the confusion matrix"},{"metadata":{"trusted":true,"_uuid":"651af7c2d26d228f2994cdd3bdd92c6a8ed5f8e8"},"cell_type":"code","source":"# Look at confusion matrix \n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63947cd1e45585e8642e3baf82c412b8335b4d88"},"cell_type":"markdown","source":"## Let's now have look at the miscalssified images"},{"metadata":{"trusted":true,"_uuid":"749807774261d972eb204b4e26f656acc23405c4"},"cell_type":"code","source":"# Display some error results \n\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1984ea187f4017a602a06b6cc6bf8599f75ab098"},"cell_type":"markdown","source":"## Now it's time to visualise the output for each layer"},{"metadata":{"trusted":true,"_uuid":"a1cf17212d15b4a2150fd34cb37d6e0bba02fd37"},"cell_type":"code","source":"# Extracts the outputs of the top 12 layers\nlayer_outputs = [layer.output for layer in model.layers[:12]] \nimage_num = 590  #Change it to visualise different images\n# Creates a model that will return these outputs, given the model input\nactivation_model = models.Model(inputs=model.input, outputs=layer_outputs) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5af191188e84ad5d82e326da63e80a373c3fbd82"},"cell_type":"code","source":"img_tensor = X_train[image_num]\nimg_tensor = np.expand_dims(img_tensor, axis = 0)\nactivations = activation_model.predict(img_tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45bb0fd23a2ec7f15e7a923abb6a9e2c007c68e0"},"cell_type":"code","source":"first_layer_activation = activations[0]\nfor l in layer_outputs:\n    print(l.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0e432fb65fb15e5c85e5222153673e6b63d2894"},"cell_type":"code","source":"g = plt.imshow(X_train[image_num][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"802d46ad8298c78763bced96a8117284e0863413"},"cell_type":"markdown","source":"let's visualise one filter of the first layer"},{"metadata":{"trusted":true,"_uuid":"9f2936543747cb59b6beb436604237ab3d36f625"},"cell_type":"code","source":"plt.matshow(first_layer_activation[0, :, :, 25], cmap='viridis')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e310fd5620d007c0289a2b26728c67a5da73d491"},"cell_type":"markdown","source":"### Now let's visualise all the filters of all the layers!!!!"},{"metadata":{"trusted":true,"_uuid":"556886bc9331c41ccd10456dc570c0704764cb27"},"cell_type":"code","source":"layer_names = []\nfor layer in model.layers[:12]:\n    layer_names.append(layer.name) # Names of the layers, so you can have them as part of your plot\n\nimages_per_row = 16\nfor layer_name, layer_activation in zip(layer_names, activations): # Displays the feature maps\n    n_features = layer_activation.shape[-1] # Number of features in the feature map\n    size = layer_activation.shape[1] #The feature map has shape (1, size, size, n_features).\n    n_cols = n_features // images_per_row # Tiles the activation channels in this matrix\n    display_grid = np.zeros((size * n_cols, images_per_row * size))\n    for col in range(n_cols): # Tiles each filter into a big horizontal grid\n        for row in range(images_per_row):\n            channel_image = layer_activation[0,\n                                             :, :,\n                                             col * images_per_row + row]\n            channel_image -= channel_image.mean() # Post-processes the feature to make it visually palatable\n            channel_image /= channel_image.std()\n            channel_image *= 64\n            channel_image += 128\n            channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n            display_grid[col * size : (col + 1) * size, # Displays the grid\n                         row * size : (row + 1) * size] = channel_image\n    scale = 1. / size\n    plt.figure(figsize=(scale * display_grid.shape[1],\n                        scale * display_grid.shape[0]))\n    plt.title(layer_name)\n    plt.grid(False)\n    plt.imshow(display_grid, aspect='auto', cmap='viridis')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8fcea8aa0d44fbbc09067a591474dfa209f8433"},"cell_type":"markdown","source":"## Predict Images "},{"metadata":{"trusted":true,"_uuid":"3a304abdc28f66b6a1a2990d20652b7e63151486"},"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2a960bf81829cbdcb9d526dec064b1b15a7a6d2"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}