{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-05T17:36:59.049341Z","iopub.status.idle":"2022-07-05T17:36:59.049704Z","shell.execute_reply.started":"2022-07-05T17:36:59.049542Z","shell.execute_reply":"2022-07-05T17:36:59.049559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.050979Z","iopub.status.idle":"2022-07-05T17:36:59.051377Z","shell.execute_reply.started":"2022-07-05T17:36:59.051204Z","shell.execute_reply":"2022-07-05T17:36:59.051223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.053356Z","iopub.status.idle":"2022-07-05T17:36:59.053757Z","shell.execute_reply.started":"2022-07-05T17:36:59.053587Z","shell.execute_reply":"2022-07-05T17:36:59.053605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.array(data)\nm, n = np.shape(data)\nnp.random.shuffle(data)\n\ndata_dev = data[0:1000].T\nY_dev = data_dev[0]\nX_dev = data_dev[1:n]\n\ndata_train = data[1000:m].T\nY_train = data_train[0]\nX_train = data_train[1:n]","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.054762Z","iopub.status.idle":"2022-07-05T17:36:59.055590Z","shell.execute_reply.started":"2022-07-05T17:36:59.055381Z","shell.execute_reply":"2022-07-05T17:36:59.055405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot(Y):\n    Y = np.array(Y)\n    one_hot_Y = np.zeros((Y.size, Y.max()+1))\n    one_hot_Y[np.arange(Y.size), Y] = 1\n    return one_hot_Y\nY_dev = one_hot(Y_dev)\nX_dev = np.reshape([X_dev[:, i] for i in range(1000)], (1000, 784, 1))\nY_dev = np.reshape([Y_dev[i] for i in range(1000)], (1000, 10, 1))","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.056778Z","iopub.status.idle":"2022-07-05T17:36:59.057174Z","shell.execute_reply.started":"2022-07-05T17:36:59.056998Z","shell.execute_reply":"2022-07-05T17:36:59.057017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dense_Layer(): # A Dense Layer is a net between two connected neural layers\n    learning_rate = 0.01\n    \n    def __init__(self, input_size, output_size, activation, activation_prime):\n        self.actv = activation\n        self.actv_prime = activation_prime\n        self.weights = np.random.randn(output_size, input_size)\n        self.biases = np.zeros((output_size, 1))\n        \n    def forward(self, X):\n        self.input = np.array(X)\n        self.output = self.actv(np.dot(self.weights, self.input) + self.biases)\n        return self.output\n    \n    def backward(self, output_grad):\n        self.weights_grad = np.dot(output_grad, self.input.T)\n        self.biases_grad = output_grad\n        self.input_grad = np.dot(self.weights.T, output_grad)\n        self.update_params()\n        return np.multiply(self.input_grad , self.actv_prime(self.input))\n        \n    def update_params(self):\n        self.weights -= self.weights_grad*self.learning_rate\n        self.biases -= self.biases_grad*self.learning_rate\n\ndef mse(Y_true, Y_pred):\n    return np.mean(np.power(Y_true - Y_pred, 2))\n\ndef mse_prime(Y_true, Y_pred):\n    return 2*(Y_pred - Y_true) / np.size(Y_true)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.058431Z","iopub.status.idle":"2022-07-05T17:36:59.058805Z","shell.execute_reply.started":"2022-07-05T17:36:59.058640Z","shell.execute_reply":"2022-07-05T17:36:59.058659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X_dev\nY = Y_dev","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.060186Z","iopub.status.idle":"2022-07-05T17:36:59.060556Z","shell.execute_reply.started":"2022-07-05T17:36:59.060381Z","shell.execute_reply":"2022-07-05T17:36:59.060399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ReLu = lambda x: np.maximum(0, x)\nReLu_prime = lambda x: x > 0\nSoftMax = lambda x: np.nan_to_num(np.exp(x))/ np.sum(np.nan_to_num(np.exp(x)))\nSoftMax_prime = lambda x: x * (1-x)\n\nDense_Layer.learning_rate = 0.01\nDNN = [\n    Dense_Layer(784, 10, ReLu, ReLu_prime),\n    Dense_Layer(10, 10, SoftMax, SoftMax_prime)\n]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.061557Z","iopub.status.idle":"2022-07-05T17:36:59.061938Z","shell.execute_reply.started":"2022-07-05T17:36:59.061744Z","shell.execute_reply":"2022-07-05T17:36:59.061761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10000\n\nfor e in range(epochs):\n    error = 0\n    for x, y in zip(X, Y):\n        output = x\n        for layer in DNN:\n            output = layer.forward(output)\n        \n        error = mse(y, output)\n        \n        grad = mse_prime(y, output)\n        for layer in reversed(DNN):\n            grad = layer.backward(grad)\n    \n    error /= len(x)\n    print(\"epochs: \",e+1,\"/\",epochs,\", error: \",error)","metadata":{"execution":{"iopub.status.busy":"2022-07-05T17:36:59.063565Z","iopub.status.idle":"2022-07-05T17:36:59.063909Z","shell.execute_reply.started":"2022-07-05T17:36:59.063737Z","shell.execute_reply":"2022-07-05T17:36:59.063753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}