{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Classification with Neural Network and Adam optimization using Numpy only <br>\n\nAuthor: jvachier <br>\nCreation date: July 2022 <br>\nPublication date: July 2022 <br>\n\nMy goal is to classify $10$ digits from $0$ to $9$ with Neural Networks using Numpy only. The training set (train.csv) contains $42 000$ labelled pictures and the testing set (test.csv) contains $28 000$ non-labelled pictures. Each picture has $784$ pixels ($28 \\times 28$) and is the input size of the network. <br>\n\nThe Network is built using two hidden layers using Adam optimization. The first layer contains $128$ neurons, the second layer $40$ neurons and the output layer $10$ neurons.\n\nIn this Network, the activation function for the intermediate layers is ReLu and the accuracy is computed. Moreover, the cost function used is the mean squared error (MSE). <br>\n\nThis Network is optimized using Adam and compared with another Network without optimization. The learning rate is given by $\\alpha$.  <br>\n\nThe Networks are first trained independently on the training set and then tested using the test set. The accuracy reaches $100\\%$ for the Network optimized with Adam.  <br>\n\nReference: <br>\n- 'ADAM: A METHOD FOR STOCHASTIC OPTIMIZATION' D.P Kingma and J. Lei Ba https://arxiv.org/pdf/1412.6980.pdf","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import to_categorical\nfrom PIL import Image as im\nfrom IPython.display import display \nfrom matplotlib.pyplot import imshow \nimport time","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.315445Z","iopub.execute_input":"2022-07-11T17:54:48.315821Z","iopub.status.idle":"2022-07-11T17:54:48.321436Z","shell.execute_reply.started":"2022-07-11T17:54:48.315791Z","shell.execute_reply":"2022-07-11T17:54:48.320597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions","metadata":{}},{"cell_type":"code","source":"## Activation Functions ##\n## ReLu ##\ndef Relu(x):\n    return np.maximum(0,x)\n    \ndef Relu_derive(x):\n    return np.where(x <= 0, 0, 1)\n    \n## Sigmoid ##\ndef Sigmoid(x):\n    return 1.0/(1.0 + np.exp(-x))\n    \ndef Sigmoid_deriv(x):\n    return Sigmoid(x)*(1.0 - Sigmoid(x))\n\n## Softmax ##\ndef Softmax(x):\n    return np.exp(x)/sum(np.exp(x))\n\n## Cost function ##\ndef cost(y_data,y_prediction):\n    return np.sum((y_data - y_prediction)**2/y_data.size)\n\ndef Categrorical_accuracy(output_layer,data): \n    maxarg = lambda x : np.argmax(x)\n    yTrue = np.array([maxarg(z) for z in data.T])\n    yPred = np.array([maxarg(z) for z in output_layer.T])\n    CategoricalAccuracy = sum(yPred == yTrue)/len(yPred)\n    return CategoricalAccuracy","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.323897Z","iopub.execute_input":"2022-07-11T17:54:48.324543Z","iopub.status.idle":"2022-07-11T17:54:48.336673Z","shell.execute_reply.started":"2022-07-11T17:54:48.324508Z","shell.execute_reply":"2022-07-11T17:54:48.335745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Networks ","metadata":{}},{"cell_type":"markdown","source":"### 3 Layers with Softmax with Adam Optimization","metadata":{}},{"cell_type":"code","source":"def Layers3_softmax_Adam(n_xtrain,y_train,iterations, learning_rate):\n    iterations    = iterations\n    learning_rate = learning_rate\n    n_neurons_layer1 = 128\n    n_neurons_layer2 = 40\n    n_neurons_layer3 = 10 # 10 numbers\n    \n    \n    ## initialization ##\n    # Random matrix and vector #\n    # Layer 1 #\n    weights_layer1 = np.random.rand(n_neurons_layer1,n_xtrain) - 0.5 #matrix drwan from uniform distrib - if randn then from Normal distribution\n    biases_layer1  = np.random.rand(n_neurons_layer1,1) - 0.5  #vector\n    # Layer 2 #\n    weights_layer2 = np.random.rand(n_neurons_layer2,n_neurons_layer1)- 0.5  #matrix\n    biases_layer2  = np.random.rand(n_neurons_layer2,1) - 0.5  #vector\n    # Layer 3 #\n    weights_layer3 = np.random.rand(n_neurons_layer3,n_neurons_layer2)- 0.5  #matrix\n    biases_layer3  = np.random.rand(n_neurons_layer3,1) - 0.5  #vector\n    \n    ## Adam optimization ##\n    ## WEIGHT ##\n    m_now_weight3  = np.zeros((n_neurons_layer3,n_neurons_layer2))\n    v_now_weight3  = m_now_weight3\n    m_now_weight2  = np.zeros((n_neurons_layer2,n_neurons_layer1))\n    v_now_weight2  = m_now_weight2   \n    m_now_weight1  = np.zeros((n_neurons_layer1,n_xtrain))\n    v_now_weight1  = m_now_weight1\n    \n    ## BIAS ##\n    B_now_bias3    = np.zeros((n_neurons_layer3,1))\n    vB_now_bias3   = B_now_bias3      \n    B_now_bias2    = np.zeros((n_neurons_layer2,1))\n    vB_now_bias2   = B_now_bias2     \n    B_now_bias1    = np.zeros((n_neurons_layer1,1))\n    vB_now_bias1   = B_now_bias1\n    \n    accuracy_history_Network3 = []\n    sqrt_mse_history_Network3 = []\n    r2_score_history_Network3 = []\n    Cost_history_Network3     = []\n    \n    for iters in range(iterations):\n        ## Forward Propagation ##\n        Z_layer1          = weights_layer1 @ x_train_np + biases_layer1\n        activation_layer1 = Relu(Z_layer1)\n        Z_layer2          = weights_layer2 @ activation_layer1 + biases_layer2\n        activation_layer2 = Relu(Z_layer2)\n        Z_layer3          = weights_layer3 @ activation_layer2 + biases_layer3\n        activation_layer3 = Softmax(Z_layer3)\n        \n        \n        ## Backpropagation using Adam Optimization ##    \n        beta1   = 0.9\n        beta2   = 0.99\n        epsilon = 1e-8 \n        \n        BETA1 = (1.0 - beta1)\n        BETA2 = (1.0 - beta2)\n        \n        invBETA1 = 1.0/BETA1\n        invBETA2 = 1.0/BETA2\n            \n        m_past_weight3 = m_now_weight3\n        v_past_weight3 = v_now_weight3\n        \n        m_past_weight2 = m_now_weight2\n        v_past_weight2 = v_now_weight2\n        \n        m_past_weight1 = m_now_weight1\n        v_past_weight1 = v_now_weight1\n\n        B_past_bias3  = B_now_bias3\n        vB_past_bias3 = vB_now_bias3\n        \n        B_past_bias2  = B_now_bias2\n        vB_past_bias2 = vB_now_bias2\n        \n        B_past_bias1  = B_now_bias1\n        vB_past_bias1 = vB_now_bias1\n\n\n        \n        ## backward propagation ##\n        MSE      = activation_layer3 - y_train \n        Layer_2b = (weights_layer3.T @ MSE) * Relu_derive(Z_layer2) \n        Layer_2w = Layer_2b @ activation_layer1.T\n        Layer_1b = (weights_layer2.T  @ Layer_2b) * Relu_derive(Z_layer1) \n        Layer_1w = Layer_1b @ x_train_np.T \n    \n\n        ## WEIGHT ##\n        m_now_weight3 = beta1*m_past_weight3 + BETA1 * (MSE @ activation_layer2.T) #Layer 3\n        m_now_weight2 = beta1*m_past_weight2 + BETA1 * Layer_2w #Layer 2\n        m_now_weight1 = beta1*m_past_weight1 + BETA1 * Layer_1w #Layer 1\n\n        v_now_weight3 = beta2*v_past_weight3 + BETA2 * (MSE @ activation_layer2.T)**2 \n        v_now_weight2 = beta2*v_past_weight2 + BETA2 * Layer_2w**2\n        v_now_weight1 = beta2*v_past_weight1 + BETA2 * Layer_1w**2\n\n        hm_now_weight3 = m_now_weight3 * invBETA1\n        hm_now_weight2 = m_now_weight2 * invBETA1\n        hm_now_weight1 = m_now_weight1 * invBETA1\n        \n        hv_now_weight3 = v_now_weight3 * invBETA2\n        hv_now_weight2 = v_now_weight2 * invBETA2\n        hv_now_weight1 = v_now_weight1 * invBETA2\n\n        \n        \n        ## BIAS ##\n        B_now_bias3 = beta1*B_past_bias3 + BETA1 * (np.sum(MSE)/len(MSE))\n        B_now_bias2 = beta1*B_past_bias2 + BETA1 * (np.sum(Layer_2b)/len(Layer_2b))\n        B_now_bias1 = beta1*B_past_bias1 + BETA1 * (np.sum(Layer_1b)/len(Layer_1b))\n\n        vB_now_bias3 = beta2*vB_past_bias3 + BETA2 * (np.sum(MSE)/len(MSE))**2 \n        vB_now_bias2 = beta2*vB_past_bias2 + BETA2 * (np.sum(Layer_2b)/len(Layer_2b))**2\n        vB_now_bias1 = beta2*vB_past_bias1 + BETA2 * (np.sum(Layer_1b)/len(Layer_1b))**2\n\n        hB_now_bias3 = B_now_bias3 * invBETA1\n        hB_now_bias2 = B_now_bias2 * invBETA1\n        hB_now_bias1 = B_now_bias1 * invBETA1\n        \n        hvB_now_bias3 = vB_now_bias3 * invBETA2\n        hvB_now_bias2 = vB_now_bias2 * invBETA2\n        hvB_now_bias1 = vB_now_bias1 * invBETA2\n\n\n        correction_weight3 = hm_now_weight3 * (learning_rate/(np.sqrt(hv_now_weight3) + epsilon))\n        correction_weight2 = hm_now_weight2 * (learning_rate/(np.sqrt(hv_now_weight2) + epsilon))\n        correction_weight1 = hm_now_weight1 * (learning_rate/(np.sqrt(hv_now_weight1) + epsilon))\n        \n        \n        correction_bias3 = hB_now_bias3 * (learning_rate/(np.sqrt(hvB_now_bias3) + epsilon))\n        correction_bias2 = hB_now_bias2 * (learning_rate/(np.sqrt(hvB_now_bias2) + epsilon))\n        correction_bias1 = hB_now_bias1 * (learning_rate/(np.sqrt(hvB_now_bias1) + epsilon))\n        \n        # Layer 3 #\n        biases_layer3  = biases_layer3  - correction_bias3\n        weights_layer3 = weights_layer3 - correction_weight3  \n        # Layer 2 #\n        weights_layer2 = weights_layer2 - correction_weight2\n        biases_layer2  = biases_layer2  - correction_bias2\n        # Layer 1 #\n        weights_layer1 = weights_layer1 - correction_weight1\n        biases_layer1  = biases_layer1  - correction_bias1\n\n        ## accurary ##\n        categoricalAccuracy = Categrorical_accuracy(activation_layer3,y_train)\n\n        accuracy_history_Network3.append(categoricalAccuracy)\n        sqrt_mse_history_Network3.append(np.sqrt(mean_squared_error(y_train,activation_layer3)))\n        r2_score_history_Network3.append(r2_score(y_train,activation_layer3))\n        Cost_history_Network3.append(cost(y_train,activation_layer3))\n\n\n        if iters%100 == 0:\n            metric = tf.keras.metrics.CategoricalAccuracy()\n            metric.update_state(y_train.T,activation_layer3.T)\n            print(\"Epoch\", iters)\n            print(\"Root Mean Squared Error: \",np.sqrt(mean_squared_error(y_train,activation_layer3)))\n            print(\"R2 Score: \",r2_score(y_train,activation_layer3))\n            print(\"Cost: \", cost(y_train,activation_layer3))\n            print(\"Categorical Accuracy: \", categoricalAccuracy, \" (Keras: \", metric.result().numpy(), \")\")\n            print(\"\\n\")\n            \n    return Cost_history_Network3, accuracy_history_Network3, sqrt_mse_history_Network3, r2_score_history_Network3, weights_layer1, biases_layer1, weights_layer2, biases_layer2, weights_layer3, biases_layer3","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.338205Z","iopub.execute_input":"2022-07-11T17:54:48.338533Z","iopub.status.idle":"2022-07-11T17:54:48.365785Z","shell.execute_reply.started":"2022-07-11T17:54:48.338504Z","shell.execute_reply":"2022-07-11T17:54:48.364960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Validation Network 3","metadata":{}},{"cell_type":"code","source":"def Layers3_softmax_Validation(x_train_np,m_ytrain,weights_layer1, biases_layer1, weights_layer2, biases_layer2, weights_layer3, biases_layer3):\n    for i in range(10):\n        a          = np.random.randint(0,m_ytrain)\n        array_test = np.reshape(x_train_np[:,a]*255.0, (28, 28))\n        data_test  = im.fromarray(array_test)\n        imshow(data_test)\n        Z_layer1          = weights_layer1 @ x_train_np + biases_layer1\n        activation_layer1 = Relu(Z_layer1)\n\n        Z_layer2          = weights_layer2 @ activation_layer1 + biases_layer2\n        activation_layer2 = Relu(Z_layer2)\n\n        Z_layer3          = weights_layer3 @ activation_layer2 + biases_layer3\n        activation_layer3 = Softmax(Z_layer3)\n        print(\"The number is: \", np.argmax(activation_layer3,0)[a])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.367374Z","iopub.execute_input":"2022-07-11T17:54:48.367849Z","iopub.status.idle":"2022-07-11T17:54:48.386215Z","shell.execute_reply.started":"2022-07-11T17:54:48.367820Z","shell.execute_reply":"2022-07-11T17:54:48.385454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3 Layers with Softmax without Adam","metadata":{}},{"cell_type":"code","source":"def Layers3_softmax(n_xtrain,y_train,iterations, learning_rate):\n    iterations    = iterations\n    learning_rate = learning_rate\n    n_neurons_layer1 = 128\n    n_neurons_layer2 = 40\n    n_neurons_layer3 = 10 # 10 numbers\n    \n    \n    ## initialization ##\n    # Random matrix and vector #\n    # Layer 1 #\n    weights_layer1 = np.random.rand(n_neurons_layer1,n_xtrain) - 0.5 #matrix drwan from uniform distrib - if randn then from Normal distribution\n    biases_layer1  = np.random.rand(n_neurons_layer1,1) - 0.5  #vector\n    # Layer 2 #\n    weights_layer2 = np.random.rand(n_neurons_layer2,n_neurons_layer1)- 0.5  #matrix\n    biases_layer2  = np.random.rand(n_neurons_layer2,1) - 0.5  #vector\n    # Layer 3 #\n    weights_layer3 = np.random.rand(n_neurons_layer3,n_neurons_layer2)- 0.5  #matrix\n    biases_layer3  = np.random.rand(n_neurons_layer3,1) - 0.5  #vector\n    \n    \n    \n    accuracy_history_Network3 = []\n    sqrt_mse_history_Network3 = []\n    r2_score_history_Network3 = []\n    Cost_history_Network3     = []\n    \n    for j in range(iterations):\n        ## Forward Propagation ##\n        Z_layer1          = weights_layer1 @ x_train_np + biases_layer1\n        activation_layer1 = Relu(Z_layer1)\n        Z_layer2          = weights_layer2 @ activation_layer1 + biases_layer2\n        activation_layer2 = Relu(Z_layer2)\n        Z_layer3          = weights_layer3 @ activation_layer2 + biases_layer3\n        activation_layer3 = Softmax(Z_layer3)\n        \n       \n        ## backward propagation ##\n        MSE      = activation_layer3 - y_train \n        Layer_2b = (weights_layer3.T @ MSE) * Relu_derive(Z_layer2) \n        Layer_2w = Layer_2b @ activation_layer1.T\n        Layer_1b = (weights_layer2.T  @ Layer_2b) * Relu_derive(Z_layer1) \n        Layer_1w = Layer_1b @ x_train_np.T \n        \n        \n        ## BIAS ##\n        # Layer 3 #\n        biases_layer3  = biases_layer3 - learning_rate* (np.sum(MSE)/len(MSE))\n        weights_layer3 = weights_layer3 - learning_rate*(MSE @ activation_layer2.T)  \n        # Layer 2 #\n        weights_layer2 = weights_layer2 - learning_rate * Layer_2w\n        biases_layer2  = biases_layer2 - learning_rate * (np.sum(Layer_2b)/len(Layer_2b))\n        # Layer 1 #\n        weights_layer1 = weights_layer1 - learning_rate * Layer_1w\n        biases_layer1  = biases_layer1 - learning_rate * (np.sum(Layer_1b)/len(Layer_1b))\n   \n\n        ## accurary ##\n        categoricalAccuracy = Categrorical_accuracy(activation_layer3,y_train)\n\n        accuracy_history_Network3.append(categoricalAccuracy)\n        sqrt_mse_history_Network3.append(np.sqrt(mean_squared_error(y_train,activation_layer3)))\n        r2_score_history_Network3.append(r2_score(y_train,activation_layer3))\n        Cost_history_Network3.append(cost(y_train,activation_layer3))\n\n\n        if j%100 == 0:\n            metric = tf.keras.metrics.CategoricalAccuracy()\n            metric.update_state(y_train.T,activation_layer3.T)\n            print(\"Epoch\", j)\n            print(\"Root Mean Squared Error: \",np.sqrt(mean_squared_error(y_train,activation_layer3)))\n            print(\"R2 Score: \",r2_score(y_train,activation_layer3))\n            print(\"Cost: \", cost(y_train,activation_layer3))\n            print(\"Categorical Accuracy: \", categoricalAccuracy, \" (Keras: \", metric.result().numpy(), \")\")\n            print(\"\\n\")\n            \n    return Cost_history_Network3, accuracy_history_Network3, sqrt_mse_history_Network3, r2_score_history_Network3, weights_layer1, biases_layer1, weights_layer2, biases_layer2, weights_layer3, biases_layer3","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.387851Z","iopub.execute_input":"2022-07-11T17:54:48.388168Z","iopub.status.idle":"2022-07-11T17:54:48.402538Z","shell.execute_reply.started":"2022-07-11T17:54:48.388138Z","shell.execute_reply":"2022-07-11T17:54:48.401530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data importation","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:48.404379Z","iopub.execute_input":"2022-07-11T17:54:48.404830Z","iopub.status.idle":"2022-07-11T17:54:52.419177Z","shell.execute_reply.started":"2022-07-11T17:54:48.404802Z","shell.execute_reply":"2022-07-11T17:54:52.418028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head() #there are labels","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.420748Z","iopub.execute_input":"2022-07-11T17:54:52.421197Z","iopub.status.idle":"2022-07-11T17:54:52.439315Z","shell.execute_reply.started":"2022-07-11T17:54:52.421164Z","shell.execute_reply":"2022-07-11T17:54:52.438111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head() # there are no labels here","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.440889Z","iopub.execute_input":"2022-07-11T17:54:52.442474Z","iopub.status.idle":"2022-07-11T17:54:52.463639Z","shell.execute_reply.started":"2022-07-11T17:54:52.442427Z","shell.execute_reply":"2022-07-11T17:54:52.462523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.T\ndf_test  = df_test.T","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.465034Z","iopub.execute_input":"2022-07-11T17:54:52.465385Z","iopub.status.idle":"2022-07-11T17:54:52.471430Z","shell.execute_reply.started":"2022-07-11T17:54:52.465350Z","shell.execute_reply":"2022-07-11T17:54:52.470173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training ","metadata":{}},{"cell_type":"code","source":"label      = df_train.iloc[0,:] #get the label\nx_train    = df_train.iloc[1:,:]/255 #normalization\nx_train_np = x_train.to_numpy()\ny_train = label.to_numpy()\ny_train = to_categorical(y_train, num_classes=10)\ny_train = y_train.T\nprint(x_train_np.shape,y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.472668Z","iopub.execute_input":"2022-07-11T17:54:52.473044Z","iopub.status.idle":"2022-07-11T17:54:52.563232Z","shell.execute_reply.started":"2022-07-11T17:54:52.473013Z","shell.execute_reply":"2022-07-11T17:54:52.562074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_xtrain, m_xtrain = x_train_np.shape\nn_ytrain, m_ytrain = y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.564872Z","iopub.execute_input":"2022-07-11T17:54:52.565549Z","iopub.status.idle":"2022-07-11T17:54:52.571751Z","shell.execute_reply.started":"2022-07-11T17:54:52.565506Z","shell.execute_reply":"2022-07-11T17:54:52.570425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Neural Network 3: 3 Layers with Softmax Adam function ","metadata":{}},{"cell_type":"code","source":"Cost_history_Network3, accuracy_history_Network3, sqrt_mse_history_Network3, r2_score_history_Network3, weights_layer1_Network3, biases_layer1_Network3, weights_layer2_Network3, biases_layer2_Network3, weights_layer3_Network3, biases_layer3_Network3 = Layers3_softmax_Adam(n_xtrain,y_train,1001,1E-2)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T17:54:52.576188Z","iopub.execute_input":"2022-07-11T17:54:52.577383Z","iopub.status.idle":"2022-07-11T18:08:47.764628Z","shell.execute_reply.started":"2022-07-11T17:54:52.577284Z","shell.execute_reply":"2022-07-11T18:08:47.763429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Validation Network 3","metadata":{}},{"cell_type":"code","source":"Layers3_softmax_Validation(x_train_np,m_ytrain,weights_layer1_Network3, biases_layer1_Network3, weights_layer2_Network3, biases_layer2_Network3, weights_layer3_Network3, biases_layer3_Network3)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:08:47.766055Z","iopub.execute_input":"2022-07-11T18:08:47.766450Z","iopub.status.idle":"2022-07-11T18:08:52.134364Z","shell.execute_reply.started":"2022-07-11T18:08:47.766418Z","shell.execute_reply":"2022-07-11T18:08:52.133145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Neural Network 3: 3 Layers with Softmax function ","metadata":{}},{"cell_type":"code","source":"Cost_history_Network3_1, accuracy_history_Network3_1, sqrt_mse_history_Network3_1, r2_score_history_Network3_1, weights_layer1_Network3_1, biases_layer1_Network3_1, weights_layer2_Network3_1, biases_layer2_Network3_1, weights_layer3_Network3_1, biases_layer3_Network3_1 = Layers3_softmax(n_xtrain,y_train,1001,1E-5) ","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:08:52.135899Z","iopub.execute_input":"2022-07-11T18:08:52.136685Z","iopub.status.idle":"2022-07-11T18:22:40.356499Z","shell.execute_reply.started":"2022-07-11T18:08:52.136640Z","shell.execute_reply":"2022-07-11T18:22:40.355294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ploting Results","metadata":{}},{"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"] = (15,10)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:40.357907Z","iopub.execute_input":"2022-07-11T18:22:40.358537Z","iopub.status.idle":"2022-07-11T18:22:40.363378Z","shell.execute_reply.started":"2022-07-11T18:22:40.358503Z","shell.execute_reply":"2022-07-11T18:22:40.362415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(accuracy_history_Network3, label=\"Network 3 Adam - $\\\\alpha = 10^{-2}$\")\nplt.plot(accuracy_history_Network3_1, label=\"Network 3 - $\\\\alpha = 10^{-5}$\")\nplt.xlabel(\"Epoch\",fontsize = 16)\nplt.ylabel(\"Accuracy\",fontsize = 16)\nplt.xticks(fontsize = 15)\nplt.yticks(fontsize = 15)\nplt.legend(fontsize = 15)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:40.365220Z","iopub.execute_input":"2022-07-11T18:22:40.365551Z","iopub.status.idle":"2022-07-11T18:22:40.643695Z","shell.execute_reply.started":"2022-07-11T18:22:40.365523Z","shell.execute_reply":"2022-07-11T18:22:40.642462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(Cost_history_Network3, label=\"Network 3 Adam\")\nplt.plot(Cost_history_Network3_1, label=\"Network 3\")\nplt.xlabel(\"Epoch\",fontsize = 16)\nplt.ylabel(\"Cost\",fontsize = 16)\nplt.xticks(fontsize = 15)\nplt.yticks(fontsize = 15)\nplt.legend(fontsize = 15)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:40.645576Z","iopub.execute_input":"2022-07-11T18:22:40.645943Z","iopub.status.idle":"2022-07-11T18:22:40.884068Z","shell.execute_reply.started":"2022-07-11T18:22:40.645911Z","shell.execute_reply":"2022-07-11T18:22:40.882926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(sqrt_mse_history_Network3, label=\"Network 3 Adam\")\nplt.plot(sqrt_mse_history_Network3_1, label=\"Network 3\")\nplt.xlabel(\"Epoch\",fontsize = 16)\nplt.ylabel(\"Root Mean Squared Error\",fontsize = 16)\nplt.xticks(fontsize = 15)\nplt.yticks(fontsize = 15)\nplt.legend(fontsize = 15)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:40.885430Z","iopub.execute_input":"2022-07-11T18:22:40.885750Z","iopub.status.idle":"2022-07-11T18:22:41.116107Z","shell.execute_reply.started":"2022-07-11T18:22:40.885721Z","shell.execute_reply":"2022-07-11T18:22:41.115265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(r2_score_history_Network3, label=\"Network 3 Adam\")\nplt.plot(r2_score_history_Network3_1, label=\"Network 3\")\nplt.xlabel(\"Epoch\",fontsize = 16)\nplt.ylabel(\"R2 Coefficient\",fontsize = 16)\nplt.xticks(fontsize = 15)\nplt.yticks(fontsize = 15)\nplt.legend(fontsize = 15)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:41.117521Z","iopub.execute_input":"2022-07-11T18:22:41.118106Z","iopub.status.idle":"2022-07-11T18:22:41.365471Z","shell.execute_reply.started":"2022-07-11T18:22:41.118070Z","shell.execute_reply":"2022-07-11T18:22:41.364567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing ","metadata":{}},{"cell_type":"code","source":"x_test         = df_test/255.\nx_test_np      = x_test.to_numpy()\nn_test, m_test = x_test_np.shape\nprint(n_test,m_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:41.366839Z","iopub.execute_input":"2022-07-11T18:22:41.367432Z","iopub.status.idle":"2022-07-11T18:22:41.414118Z","shell.execute_reply.started":"2022-07-11T18:22:41.367400Z","shell.execute_reply":"2022-07-11T18:22:41.412414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Network 3","metadata":{}},{"cell_type":"code","source":"for i in range(10):\n    a          = np.random.randint(0,m_test)\n    array_test = np.reshape(x_test_np[:,a]*255.0, (28, 28))\n    data_test  = im.fromarray(array_test)\n    imshow(data_test)\n    Z_layer1          = weights_layer1_Network3 @ x_test_np[:,a,None] + biases_layer1_Network3\n    activation_layer1 = Relu(Z_layer1)\n    \n    Z_layer2          = weights_layer2_Network3 @ activation_layer1 + biases_layer2_Network3\n    activation_layer2 = Relu(Z_layer2)\n    \n    Z_layer3          = weights_layer3_Network3 @ activation_layer2 + biases_layer3_Network3\n    activation_layer3 = Softmax(Z_layer3)\n    print(\"The number is: \", np.argmax(activation_layer3,0))\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:41.415541Z","iopub.execute_input":"2022-07-11T18:22:41.415879Z","iopub.status.idle":"2022-07-11T18:22:43.336359Z","shell.execute_reply.started":"2022-07-11T18:22:41.415849Z","shell.execute_reply":"2022-07-11T18:22:43.335417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):\n    a          = np.random.randint(0,m_test)\n    array_test = np.reshape(x_test_np[:,a]*255.0, (28, 28))\n    data_test  = im.fromarray(array_test)\n    imshow(data_test)\n    Z_layer1          = weights_layer1_Network3 @ x_test_np[:,a,None] + biases_layer1_Network3[:,0,None]\n    activation_layer1 = Relu(Z_layer1)\n    \n    Z_layer2          = weights_layer2_Network3 @ activation_layer1 + biases_layer2_Network3[:,0,None]\n    activation_layer2 = Relu(Z_layer2)\n    \n    Z_layer3          = weights_layer3_Network3 @ activation_layer2 + biases_layer3_Network3[:,0,None]\n    activation_layer3 = Softmax(Z_layer3)\n    print(\"The number is: \", np.argmax(activation_layer3,0))\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:43.337513Z","iopub.execute_input":"2022-07-11T18:22:43.337960Z","iopub.status.idle":"2022-07-11T18:22:45.635865Z","shell.execute_reply.started":"2022-07-11T18:22:43.337931Z","shell.execute_reply":"2022-07-11T18:22:45.634665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"ImageId = []\nLabel   = []\nfor i in range(m_test):\n    Z_layer1          = weights_layer1_Network3 @ x_test_np[:,i,None] + biases_layer1_Network3\n    activation_layer1 = Relu(Z_layer1)\n    \n    Z_layer2          = weights_layer2_Network3 @ activation_layer1 + biases_layer2_Network3\n    activation_layer2 = Relu(Z_layer2)\n    \n    Z_layer3          = weights_layer3_Network3 @ activation_layer2 + biases_layer3_Network3\n    activation_layer3 = Softmax(Z_layer3)\n    \n    ImageId.append(i)\n    Label.append(np.argmax(activation_layer3,0))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:45.637131Z","iopub.execute_input":"2022-07-11T18:22:45.637456Z","iopub.status.idle":"2022-07-11T18:22:47.929210Z","shell.execute_reply.started":"2022-07-11T18:22:45.637428Z","shell.execute_reply":"2022-07-11T18:22:47.927885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ImageId_array = np.asarray(ImageId)\nLabel_array   = np.asarray(Label).reshape(-1)\nprint(ImageId_array.shape,Label_array.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:47.931302Z","iopub.execute_input":"2022-07-11T18:22:47.932283Z","iopub.status.idle":"2022-07-11T18:22:47.966683Z","shell.execute_reply.started":"2022-07-11T18:22:47.932221Z","shell.execute_reply":"2022-07-11T18:22:47.965382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Prediction_dataframe = pd.DataFrame({'ImageId': ImageId_array + 1, 'Label': Label_array})\nPrediction_dataframe.to_csv('prediction_digits_adam.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:47.968885Z","iopub.execute_input":"2022-07-11T18:22:47.969890Z","iopub.status.idle":"2022-07-11T18:22:48.042902Z","shell.execute_reply.started":"2022-07-11T18:22:47.969837Z","shell.execute_reply":"2022-07-11T18:22:48.041348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10):    \n    array_test = np.reshape(x_test_np[:,i]*255.0, (28, 28))\n    data_test  = im.fromarray(array_test)\n    imshow(data_test)\n    Z_layer1          = weights_layer1_Network3 @ x_test_np[:,i,None] + biases_layer1_Network3\n    activation_layer1 = Relu(Z_layer1)\n\n    Z_layer2          = weights_layer2_Network3 @ activation_layer1 + biases_layer2_Network3\n    activation_layer2 = Relu(Z_layer2)\n\n    Z_layer3          = weights_layer3_Network3 @ activation_layer2 + biases_layer3_Network3\n    activation_layer3 = Softmax(Z_layer3)\n    print(\"The number is: \", np.argmax(activation_layer3,0))\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T18:22:48.045147Z","iopub.execute_input":"2022-07-11T18:22:48.046268Z","iopub.status.idle":"2022-07-11T18:22:49.942652Z","shell.execute_reply.started":"2022-07-11T18:22:48.046211Z","shell.execute_reply":"2022-07-11T18:22:49.941398Z"},"trusted":true},"execution_count":null,"outputs":[]}]}