{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## CNN with <250k parameters\n### Imports\n#### I'm only using basic libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-01T19:07:09.765815Z","iopub.execute_input":"2022-08-01T19:07:09.766177Z","iopub.status.idle":"2022-08-01T19:07:09.771463Z","shell.execute_reply.started":"2022-08-01T19:07:09.766132Z","shell.execute_reply":"2022-08-01T19:07:09.770211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setting the seed\n#### This ensures reproducible results","metadata":{}},{"cell_type":"code","source":"tf.random.set_seed(13)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:09.886864Z","iopub.execute_input":"2022-08-01T19:07:09.887136Z","iopub.status.idle":"2022-08-01T19:07:09.900193Z","shell.execute_reply.started":"2022-08-01T19:07:09.887111Z","shell.execute_reply":"2022-08-01T19:07:09.899072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get the Kaggle provided data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('../input/digit-recognizer/test.csv')\nY = train_df['label']\nX = train_df.drop('label', axis = 1)\ndel train_df\ntest = np.array(test_df).reshape(-1, 28, 28, 1) / 255.0\ndel test_df\nprint(np.shape(X))\nprint(np.shape(test))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:10.014745Z","iopub.execute_input":"2022-08-01T19:07:10.015000Z","iopub.status.idle":"2022-08-01T19:07:13.531418Z","shell.execute_reply.started":"2022-08-01T19:07:10.014976Z","shell.execute_reply":"2022-08-01T19:07:13.530336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get more data from the MNIST database\n#### MNIST stands for 'Modified National Institute of Standards and Technology', and the MNIST database is a large, public, database of handwritten digits","metadata":{}},{"cell_type":"code","source":"(x_train_more, y_train_more), (x_test_more, y_test_more) = tf.keras.datasets.mnist.load_data()\n\ntrain_more = np.concatenate([x_train_more, x_test_more], axis=0)\ny_train_more = np.concatenate([y_train_more, y_test_more], axis=0)\n\nY_train_tf = y_train_more\nX_train_tf = train_more.reshape(-1, 28*28)\nprint(X_train_tf.shape, Y_train_tf.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:13.533313Z","iopub.execute_input":"2022-08-01T19:07:13.534273Z","iopub.status.idle":"2022-08-01T19:07:13.818383Z","shell.execute_reply.started":"2022-08-01T19:07:13.534232Z","shell.execute_reply":"2022-08-01T19:07:13.817281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Merge the Kaggle and MNIST data and then scale it down by 255\n#### This normalizes the values to the range [0,1] and makes training faster and easier","metadata":{}},{"cell_type":"code","source":"X = np.concatenate((X, X_train_tf)).reshape(-1,28,28,1) / 255.0\nY = np.concatenate((Y, Y_train_tf))\nprint(X.shape, Y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:13.819924Z","iopub.execute_input":"2022-08-01T19:07:13.820300Z","iopub.status.idle":"2022-08-01T19:07:14.314063Z","shell.execute_reply.started":"2022-08-01T19:07:13.820265Z","shell.execute_reply":"2022-08-01T19:07:14.312971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### I am using ImageDataGenerator because I was messing around with data augmentation. This proved to be unnecessary.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ngen = ImageDataGenerator(validation_split = 0.2)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:14.316675Z","iopub.execute_input":"2022-08-01T19:07:14.317246Z","iopub.status.idle":"2022-08-01T19:07:14.322504Z","shell.execute_reply.started":"2022-08-01T19:07:14.317209Z","shell.execute_reply":"2022-08-01T19:07:14.321202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Helper function prints the graphs of training\\val loss and accuracy so I can more easily discern whether there was any point in training the model for more epochs","metadata":{}},{"cell_type":"code","source":"def Charts(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'val'], loc='upper left')\n    plt.show()\n    \n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'val'], loc='upper left')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:14.324166Z","iopub.execute_input":"2022-08-01T19:07:14.324854Z","iopub.status.idle":"2022-08-01T19:07:14.333719Z","shell.execute_reply.started":"2022-08-01T19:07:14.324816Z","shell.execute_reply":"2022-08-01T19:07:14.332771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating the model\n#### We use 3 sets of Conv2D with kernel size 3, MaxPool2D with pool size 2, and BatchNormalization, then we flatten the last BatchNormalizations output and add one more hidden dense layer.\n#### We apply dropout after the flattening, and after the hidden dense layer to further reduce variance.\n#### The setup of multiple Conv layers, then a Pool layer followed by a BatchNorm layer is somewhat of a standard practice (at least thats my impression after spending some time googling about the subject)","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, BatchNormalization, Dense, Dropout, Flatten\nfrom tensorflow.keras.optimizers import Adam\nmodel = Sequential()\n\nmodel.add(Conv2D(32, 3, padding = 'same', kernel_initializer = 'he_normal', activation = 'relu', input_shape = (28,28,1)))\nmodel.add(MaxPool2D(2))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(64, 3, padding = 'same', kernel_initializer = 'he_normal', activation = 'relu'))\nmodel.add(MaxPool2D(2))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(128, 3, padding = 'same', kernel_initializer = 'he_normal', activation = 'relu'))\nmodel.add(MaxPool2D(2))\nmodel.add(BatchNormalization())\n\nmodel.add(Flatten())\nmodel.add(Dropout(0.3))\n\nmodel.add(Dense(128, kernel_initializer = 'he_normal', activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(10, activation = 'softmax'))\n\nmodel.summary()\nmodel.compile(optimizer = Adam(learning_rate = 3e-4), loss = 'sparse_categorical_crossentropy' , metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:10:52.749054Z","iopub.execute_input":"2022-08-01T19:10:52.749732Z","iopub.status.idle":"2022-08-01T19:10:52.845301Z","shell.execute_reply.started":"2022-08-01T19:10:52.749695Z","shell.execute_reply":"2022-08-01T19:10:52.844198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train the model\n#### We are adding an on_epoch_end callback, so that when we reach a high enough accuracy on the validation set, we stop training","metadata":{}},{"cell_type":"code","source":"class CustomCallbacks(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs = None):\n        acc = logs['accuracy']\n        val_acc = logs['val_accuracy']\n        if acc >= 0.999 and val_acc >= 0.99995:\n            self.model.stop_training = True","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:10:54.256585Z","iopub.execute_input":"2022-08-01T19:10:54.257459Z","iopub.status.idle":"2022-08-01T19:10:54.263373Z","shell.execute_reply.started":"2022-08-01T19:10:54.257416Z","shell.execute_reply":"2022-08-01T19:10:54.262423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We train the model for 200 epochs with batch size 3200, which is the highest divisor of the number of samples that fits into memory","metadata":{}},{"cell_type":"code","source":"history = model.fit(gen.flow(X, y = Y, subset = 'training', batch_size = 1600), validation_data=gen.flow(X, y = Y, subset = 'validation', batch_size = 1600),\n                    epochs = 200, verbose = 1, callbacks = [CustomCallbacks()])\nCharts(history)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:10:55.467535Z","iopub.execute_input":"2022-08-01T19:10:55.468546Z","iopub.status.idle":"2022-08-01T19:13:07.598633Z","shell.execute_reply.started":"2022-08-01T19:10:55.468489Z","shell.execute_reply":"2022-08-01T19:13:07.597186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We make the final submission","metadata":{}},{"cell_type":"code","source":"pred = model.predict(test)\nprint(pred.shape)\npred = tf.math.argmax(pred, axis = -1)\nprint(pred.shape)\npred = pd.Series(pred, name='Label')\npred.head()\n\nimage_id = pd.Series(range(1,28001),name='ImageId')\nimage_id.isnull().sum()\nimage_id.shape\n\npred = pd.concat([image_id,pred],axis=1)\npred.head()\npred.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T19:07:52.304437Z","iopub.status.idle":"2022-08-01T19:07:52.305339Z","shell.execute_reply.started":"2022-08-01T19:07:52.304997Z","shell.execute_reply":"2022-08-01T19:07:52.305036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}