{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow\nfrom tensorflow.keras.layers import Conv2D, Flatten, Dropout, MaxPool2D, BatchNormalization, Reshape, Dense\nfrom tensorflow.keras import Sequential\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:17:36.399756Z","iopub.execute_input":"2022-07-04T19:17:36.400082Z","iopub.status.idle":"2022-07-04T19:17:36.408084Z","shell.execute_reply.started":"2022-07-04T19:17:36.40005Z","shell.execute_reply":"2022-07-04T19:17:36.407094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Loading Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:17:42.148894Z","iopub.execute_input":"2022-07-04T19:17:42.149221Z","iopub.status.idle":"2022-07-04T19:17:46.573522Z","shell.execute_reply.started":"2022-07-04T19:17:42.14919Z","shell.execute_reply":"2022-07-04T19:17:46.572469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Normalizing Data","metadata":{}},{"cell_type":"code","source":"X_train = train.drop(['label'],axis=1)\nX_train = X_train/255.0\nX_train = X_train.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:29.913706Z","iopub.execute_input":"2022-07-04T19:14:29.913994Z","iopub.status.idle":"2022-07-04T19:14:30.134823Z","shell.execute_reply.started":"2022-07-04T19:14:29.913936Z","shell.execute_reply":"2022-07-04T19:14:30.133922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test/255.0\ntest = test.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.136832Z","iopub.execute_input":"2022-07-04T19:14:30.137174Z","iopub.status.idle":"2022-07-04T19:14:30.181481Z","shell.execute_reply.started":"2022-07-04T19:14:30.137131Z","shell.execute_reply":"2022-07-04T19:14:30.180653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Label Encoding\nSince the labelling is from [0,9] and since the values are not ordinal, we one hot encode them","metadata":{}},{"cell_type":"code","source":"y_train = tensorflow.keras.utils.to_categorical(train['label'], num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:18:03.498544Z","iopub.execute_input":"2022-07-04T19:18:03.499144Z","iopub.status.idle":"2022-07-04T19:18:03.506107Z","shell.execute_reply.started":"2022-07-04T19:18:03.499108Z","shell.execute_reply":"2022-07-04T19:18:03.505346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading MNIST Dataset to increase size of our training dataset","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.datasets import mnist\n(X_train_MNIST, y_train_MNIST), (X_test_MNIST, y_test_MNIST) = mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.548452Z","iopub.status.idle":"2022-07-04T19:14:30.548768Z","shell.execute_reply.started":"2022-07-04T19:14:30.548606Z","shell.execute_reply":"2022-07-04T19:14:30.548622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train_MNIST.shape)\nprint(y_train_MNIST.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.55024Z","iopub.status.idle":"2022-07-04T19:14:30.550586Z","shell.execute_reply.started":"2022-07-04T19:14:30.550425Z","shell.execute_reply":"2022-07-04T19:14:30.550441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mnist = np.concatenate((X_train_MNIST, X_test_MNIST))\ntest_mnist = np.concatenate((y_train_MNIST, y_test_MNIST))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.551615Z","iopub.status.idle":"2022-07-04T19:14:30.552123Z","shell.execute_reply.started":"2022-07-04T19:14:30.551849Z","shell.execute_reply":"2022-07-04T19:14:30.551873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_mnist.shape)\nprint(test_mnist.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.553213Z","iopub.status.idle":"2022-07-04T19:14:30.553533Z","shell.execute_reply.started":"2022-07-04T19:14:30.553364Z","shell.execute_reply":"2022-07-04T19:14:30.553384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mnist = train_mnist.reshape(-1, 28, 28, 1)\ntrain_mnist = train_mnist/255.0","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.555277Z","iopub.status.idle":"2022-07-04T19:14:30.555778Z","shell.execute_reply.started":"2022-07-04T19:14:30.555595Z","shell.execute_reply":"2022-07-04T19:14:30.555616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_mnist = tensorflow.keras.utils.to_categorical(test_mnist, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.556872Z","iopub.status.idle":"2022-07-04T19:14:30.557239Z","shell.execute_reply.started":"2022-07-04T19:14:30.557056Z","shell.execute_reply":"2022-07-04T19:14:30.55708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Concatenating both datasets","metadata":{}},{"cell_type":"code","source":"images = np.concatenate((X_train, train_mnist))\nlabels = np.concatenate((y_train, test_mnist))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.558287Z","iopub.status.idle":"2022-07-04T19:14:30.558648Z","shell.execute_reply.started":"2022-07-04T19:14:30.558445Z","shell.execute_reply":"2022-07-04T19:14:30.558488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(images.shape)\nprint(labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.559857Z","iopub.status.idle":"2022-07-04T19:14:30.560217Z","shell.execute_reply.started":"2022-07-04T19:14:30.560035Z","shell.execute_reply":"2022-07-04T19:14:30.560057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation\n\nAdds rotation, zoom, horizontal movement to the present images, creating an even larger dataset, prevents overfitting and is a very strong tool","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.561231Z","iopub.status.idle":"2022-07-04T19:14:30.561548Z","shell.execute_reply.started":"2022-07-04T19:14:30.561378Z","shell.execute_reply":"2022-07-04T19:14:30.561399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, shear_range=15, zoom_range=0.1, validation_split=0.25)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.563657Z","iopub.status.idle":"2022-07-04T19:14:30.564022Z","shell.execute_reply.started":"2022-07-04T19:14:30.563822Z","shell.execute_reply":"2022-07-04T19:14:30.563841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = datagen.flow(images, labels, batch_size=256, subset='training')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.565177Z","iopub.status.idle":"2022-07-04T19:14:30.565506Z","shell.execute_reply.started":"2022-07-04T19:14:30.565333Z","shell.execute_reply":"2022-07-04T19:14:30.565354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = datagen.flow(images, labels, batch_size=64, subset='validation')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.566861Z","iopub.status.idle":"2022-07-04T19:14:30.567217Z","shell.execute_reply.started":"2022-07-04T19:14:30.56704Z","shell.execute_reply":"2022-07-04T19:14:30.567061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CNN Model","metadata":{}},{"cell_type":"code","source":"model = Sequential()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.568411Z","iopub.status.idle":"2022-07-04T19:14:30.568757Z","shell.execute_reply.started":"2022-07-04T19:14:30.568578Z","shell.execute_reply":"2022-07-04T19:14:30.568605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.add(Conv2D(filters=32, kernel_size=5, activation='relu', padding='same', input_shape=(28, 28, 1)))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Conv2D(filters=64, kernel_size=3, activation='relu', padding='same'))\nmodel.add(Conv2D(filters=64, kernel_size=3, activation='relu', padding='same'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Conv2D(filters=64, kernel_size=3, activation='relu', padding='same'))\nmodel.add(Conv2D(filters=64, kernel_size=3, activation='relu', padding='same'))\nmodel.add(MaxPool2D(pool_size=2))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.570011Z","iopub.status.idle":"2022-07-04T19:14:30.570352Z","shell.execute_reply.started":"2022-07-04T19:14:30.570175Z","shell.execute_reply":"2022-07-04T19:14:30.570198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.25))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.25))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dense(10, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.571794Z","iopub.status.idle":"2022-07-04T19:14:30.572196Z","shell.execute_reply.started":"2022-07-04T19:14:30.571995Z","shell.execute_reply":"2022-07-04T19:14:30.57202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.573526Z","iopub.status.idle":"2022-07-04T19:14:30.573898Z","shell.execute_reply.started":"2022-07-04T19:14:30.573722Z","shell.execute_reply":"2022-07-04T19:14:30.573743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint\nreduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, min_lr=0.000001)\ncheckpoint = ModelCheckpoint(filepath='model.hdf5', monitor='val_loss', save_best_only=True, save_weights_only=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.575948Z","iopub.status.idle":"2022-07-04T19:14:30.576295Z","shell.execute_reply.started":"2022-07-04T19:14:30.576132Z","shell.execute_reply":"2022-07-04T19:14:30.576149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit_generator(train_dataset, epochs=70, validation_data=test_dataset, callbacks=[reduce_lr, checkpoint])","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.576996Z","iopub.status.idle":"2022-07-04T19:14:30.57745Z","shell.execute_reply.started":"2022-07-04T19:14:30.577148Z","shell.execute_reply":"2022-07-04T19:14:30.577169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.578487Z","iopub.status.idle":"2022-07-04T19:14:30.578898Z","shell.execute_reply.started":"2022-07-04T19:14:30.57871Z","shell.execute_reply":"2022-07-04T19:14:30.578739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Predicting Values","metadata":{}},{"cell_type":"code","source":"ans = tensorflow.keras.backend.argmax(model.predict(test))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.580323Z","iopub.status.idle":"2022-07-04T19:14:30.580752Z","shell.execute_reply.started":"2022-07-04T19:14:30.580487Z","shell.execute_reply":"2022-07-04T19:14:30.58058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.DataFrame({'ImageId': range(1, len(ans)+1), 'label': ans})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T19:14:30.581669Z","iopub.status.idle":"2022-07-04T19:14:30.582027Z","shell.execute_reply.started":"2022-07-04T19:14:30.58182Z","shell.execute_reply":"2022-07-04T19:14:30.581842Z"},"trusted":true},"execution_count":null,"outputs":[]}]}