{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n \n
","metadata":{}},{"cell_type":"markdown","source":"

\n
Data\n \n
\n

","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport math","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/input/digit-recognizer/\"\ndf = pd.read_csv(path + \"train.csv\")\ndf = shuffle(df)\ndf.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = df.iloc[:,1:].values.reshape(len(df), 28, 28)\nlabels = df['label'].values\n\nX_train, X_test, y_train, y_test = train_test_split(\n features, labels, test_size=0.33, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n
Implementation\n \n
\n

","metadata":{}},{"cell_type":"markdown","source":"### Convolution 3x3","metadata":{}},{"cell_type":"code","source":"class Conv3x3:\n '''\n A Convolution layer using 3x3 filters.\n '''\n\n def __init__(self, num_filters):\n self.num_filters = num_filters\n\n # filters is a 3d array with dimensions (num_filters, 3, 3)\n # We divide by 9 to reduce the variance of our initial values\n self.filters = np.random.randn(num_filters, 3, 3) / 9\n\n def iterate_regions(self, image):\n '''\n Generates all possible 3x3 image regions using valid padding.\n - image is a 2d numpy array.\n '''\n h, w = image.shape\n\n for i in range(h - 2):\n for j in range(w - 2):\n im_region = image[i:(i + 3), j:(j + 3)]\n yield im_region, i, j\n\n def forward(self, input):\n '''\n Performs a forward pass of the conv layer using the given input.\n Returns a 3d numpy array with dimensions (h, w, num_filters).\n - input is a 2d numpy array\n '''\n self.last_input = input\n\n h, w = input.shape\n output = np.zeros((h - 2, w - 2, self.num_filters))\n\n for im_region, i, j in self.iterate_regions(input):\n output[i, j] = np.sum(im_region * self.filters, axis=(1, 2))\n\n return output\n\n def backprop(self, d_L_d_out, learn_rate):\n '''\n Performs a backward pass of the conv layer.\n - d_L_d_out is the loss gradient for this layer's outputs.\n - learn_rate is a float.\n '''\n d_L_d_filters = np.zeros(self.filters.shape)\n\n for im_region, i, j in self.iterate_regions(self.last_input):\n for f in range(self.num_filters):\n d_L_d_filters[f] += d_L_d_out[i, j, f] * im_region\n\n # Update filters\n self.filters -= learn_rate * d_L_d_filters\n\n # We aren't returning anything here since we use Conv3x3 as the first layer in our CNN.\n # Otherwise, we'd need to return the loss gradient for this layer's inputs, just like every\n # other layer in our CNN.\n return None\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MaxPool 2x2","metadata":{}},{"cell_type":"code","source":"class MaxPool2:\n '''\n A Max Pooling layer using a pool size of 2\n '''\n\n def iterate_regions(self, image):\n '''\n Generates non-overlapping 2x2 image regions to pool over.\n - image is a 2d numpy array\n '''\n h, w, _ = image.shape\n new_h = h // 2\n new_w = w // 2\n\n for i in range(new_h):\n for j in range(new_w):\n im_region = image[(i * 2):(i * 2 + 2), (j * 2):(j * 2 + 2)]\n yield im_region, i, j\n\n def forward(self, input):\n '''\n Performs a forward pass of the maxpool layer using the given input.\n Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters).\n - input is a 3d numpy array with dimensions (h, w, num_filters)\n '''\n self.last_input = input\n\n h, w, num_filters = input.shape\n output = np.zeros((h // 2, w // 2, num_filters))\n\n for im_region, i, j in self.iterate_regions(input):\n output[i, j] = np.amax(im_region, axis=(0, 1))\n\n return output\n\n def backprop(self, d_L_d_out):\n '''\n Performs a backward pass of the maxpool layer.\n Returns the loss gradient for this layer's inputs.\n - d_L_d_out is the loss gradient for this layer's outputs.\n '''\n d_L_d_input = np.zeros(self.last_input.shape)\n\n for im_region, i, j in self.iterate_regions(self.last_input):\n h, w, f = im_region.shape\n amax = np.amax(im_region, axis=(0, 1))\n\n for i2 in range(h):\n for j2 in range(w):\n for f2 in range(f):\n # If this pixel was the max value, copy the gradient to it.\n if im_region[i2, j2, f2] == amax[f2]:\n d_L_d_input[i * 2 + i2, j * 2 + j2, f2] = d_L_d_out[i, j, f2]\n \n return d_L_d_input","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Write the Softmax class","metadata":{}},{"cell_type":"code","source":"class Softmax:\n '''\n A standard fully-connected layer with softmax activation.\n '''\n\n def __init__(self, input_len, nodes):\n # We divide by input_len to reduce the variance of our initial values\n self.weights = np.random.randn(input_len, nodes) / input_len\n self.biases = np.zeros(nodes)\n\n def forward(self, input):\n '''\n Performs a forward pass of the softmax layer using the given input.\n Returns a 1d numpy array containing the respective probability values.\n - input can be any array with any dimensions.\n '''\n self.last_input_shape = input.shape\n\n input = input.flatten()\n self.last_input = input\n\n input_len, nodes = self.weights.shape\n\n totals = np.dot(input, self.weights) + self.biases\n self.last_totals = totals\n\n exp = np.exp(totals)\n return exp / np.sum(exp, axis=0)\n\n def backprop(self, d_L_d_out, learn_rate):\n '''\n Performs a backward pass of the softmax layer.\n Returns the loss gradient for this layer's inputs.\n - d_L_d_out is the loss gradient for this layer's outputs.\n - learn_rate is a float.\n '''\n # We know only 1 element of d_L_d_out will be nonzero\n for i, gradient in enumerate(d_L_d_out):\n if gradient == 0:\n continue\n\n # e^totals\n t_exp = np.exp(self.last_totals)\n\n # Sum of all e^totals\n S = np.sum(t_exp)\n\n # Gradients of out[i] against totals\n d_out_d_t = -t_exp[i] * t_exp / (S ** 2)\n d_out_d_t[i] = t_exp[i] * (S - t_exp[i]) / (S ** 2)\n\n # Gradients of totals against weights/biases/input\n d_t_d_w = self.last_input\n d_t_d_b = 1\n d_t_d_inputs = self.weights\n\n # Gradients of loss against totals\n d_L_d_t = gradient * d_out_d_t\n\n # Gradients of loss against weights/biases/input\n d_L_d_w = d_t_d_w[np.newaxis].T @ d_L_d_t[np.newaxis]\n d_L_d_b = d_L_d_t * d_t_d_b\n d_L_d_inputs = d_t_d_inputs @ d_L_d_t\n\n # Update weights / biases\n self.weights -= learn_rate * d_L_d_w\n self.biases -= learn_rate * d_L_d_b\n\n return d_L_d_inputs.reshape(self.last_input_shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n
Training\n \n
\n

","metadata":{}},{"cell_type":"code","source":"def forward(image, label):\n '''\n Completes a forward pass of the CNN and calculates the accuracy and\n cross-entropy loss.\n - image is a 2d numpy array\n - label is a digit\n '''\n # We transform the image from [0, 255] to [-0.5, 0.5] to make it easier\n # to work with. This is standard practice.\n out = conv.forward((image / 255) - 0.5)\n out = pool.forward(out)\n out = softmax.forward(out)\n\n # Calculate cross-entropy loss and accuracy. np.log() is the natural log.\n loss = -np.log(out[label])\n acc = 1 if np.argmax(out) == label else 0\n\n return out, loss, acc\n\ndef train(im, label, lr=.005):\n '''\n Completes a full training step on the given image and label.\n Returns the cross-entropy loss and accuracy.\n - image is a 2d numpy array\n - label is a digit\n - lr is the learning rate\n '''\n # Forward\n out, loss, acc = forward(im, label)\n\n # Calculate initial gradient\n gradient = np.zeros(10)\n gradient[label] = -1 / out[label]\n\n # Backprop\n gradient = softmax.backprop(gradient, lr)\n gradient = pool.backprop(gradient)\n gradient = conv.backprop(gradient, lr)\n\n return loss, acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 28x28x1 -> 26x26x8\nconv = Conv3x3(8)\n\n# 26x26x8 -> 13x13x8\npool = MaxPool2()\n\n# 13x13x8 -> 10\nsoftmax = Softmax(13 * 13 * 8, 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(1):\n print('--- Epoch %d ---' % (epoch + 1))\n\n # shuffle\n X_train, y_train = shuffle(X_train, y_train)\n\n loss = 0\n num_correct = 0\n for i, (im, label) in enumerate(zip(X_train, y_train)):\n if i > 0 and i % 2000 == 1999:\n print('[Step {:5d}] Past 2000 steps: Average Loss {:.3f} | Accuracy: {:2.2f}%'\n .format(i + 1, loss / 2000, num_correct / 20))\n \n loss = 0\n num_correct = 0\n\n l, acc = train(im, label)\n loss += l\n num_correct += acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('--- Testing the CNN ---')\nloss = 0\nnum_correct = 0\nfor im, label in zip(X_test, y_test):\n _, l, acc = forward(im, label)\n loss += l\n num_correct += acc\n\nnum_tests = len(X_test)\nprint('Test Loss:', loss / num_tests)\nprint('Test Accuracy:', num_correct / num_tests)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n
Prediction\n \n
\n

","metadata":{}},{"cell_type":"code","source":"def predict(x):\n '''\n Function to predict label\n return the softmax for each\n '''\n out, _, _ = forward(im, label)\n return out","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

\n
Reference\n \n
\n

","metadata":{}},{"cell_type":"markdown","source":"I have removed my old code and added [Victor Zhou code](https://victorzhou.com/blog/intro-to-cnns-part-1/) implementation just because it's a beauty.","metadata":{}},{"cell_type":"markdown","source":"

\n
Submission\n \n
\n

","metadata":{}},{"cell_type":"code","source":"# load submission data\ndf_test = pd.read_csv(path + \"test.csv\")\nsubm_arr = df_test.values.reshape(len(df_test), 28, 28)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def subm_predict(subm_arr):\n '''\n Function to predict submission prediction\n return the max for each prediction\n '''\n result = []\n\n for i in tqdm(range(len(subm_arr))):\n result.append([i + 1, np.argmax(predict(subm_arr[i]))])\n \n return result\n\n# predict submission data\nresult = subm_predict(subm_arr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final = pd.DataFrame(data=result, columns=[\"ImageId\", \"Label\"])\ndf_final.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}