{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-success\">\n    <h1 align=\"center\">Digit Recognizer Competition</h1>\n    <h3 align=\"center\">Deep Learning Tutorial</h3>\n    <h4 align=\"center\"><a href=\"http://www.iran-machinelearning.ir\">Soheil Tehranipour</a></h5>\n</div>","metadata":{}},{"cell_type":"markdown","source":"<center><img src=\"https://media-exp2.licdn.com/dms/image/C4D12AQHAia5q8RDExg/article-cover_image-shrink_600_2000/0/1600202596769?e=1662595200&v=beta&t=cry4Vbh-xq8dPOSMladMps-5W0yApNHrpMioTyJzsJs\" width=70%></center>","metadata":{}},{"cell_type":"code","source":"# Some basics importations\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom keras.layers import Dense , BatchNormalization\nfrom keras.models import Sequential\nfrom keras.callbacks import ModelCheckpoint , EarlyStopping\nfrom IPython.display import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T03:05:54.198141Z","iopub.execute_input":"2022-07-15T03:05:54.198589Z","iopub.status.idle":"2022-07-15T03:06:03.002959Z","shell.execute_reply.started":"2022-07-15T03:05:54.198495Z","shell.execute_reply":"2022-07-15T03:06:03.001796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:03.004889Z","iopub.execute_input":"2022-07-15T03:06:03.005439Z","iopub.status.idle":"2022-07-15T03:06:07.699341Z","shell.execute_reply.started":"2022-07-15T03:06:03.005404Z","shell.execute_reply":"2022-07-15T03:06:07.698217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.700951Z","iopub.execute_input":"2022-07-15T03:06:07.701276Z","iopub.status.idle":"2022-07-15T03:06:07.710058Z","shell.execute_reply.started":"2022-07-15T03:06:07.701245Z","shell.execute_reply":"2022-07-15T03:06:07.708877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observing some training data\n\ntrain.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.712128Z","iopub.execute_input":"2022-07-15T03:06:07.712430Z","iopub.status.idle":"2022-07-15T03:06:07.755317Z","shell.execute_reply.started":"2022-07-15T03:06:07.712403Z","shell.execute_reply":"2022-07-15T03:06:07.754246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for missing values\n\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.756627Z","iopub.execute_input":"2022-07-15T03:06:07.757065Z","iopub.status.idle":"2022-07-15T03:06:07.808402Z","shell.execute_reply.started":"2022-07-15T03:06:07.757030Z","shell.execute_reply":"2022-07-15T03:06:07.807307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Total missing values\n\nsum(train.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.809742Z","iopub.execute_input":"2022-07-15T03:06:07.810536Z","iopub.status.idle":"2022-07-15T03:06:07.860399Z","shell.execute_reply.started":"2022-07-15T03:06:07.810491Z","shell.execute_reply":"2022-07-15T03:06:07.859190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = np.array(train['label'])\nX_train = train.drop('label' , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.861708Z","iopub.execute_input":"2022-07-15T03:06:07.862462Z","iopub.status.idle":"2022-07-15T03:06:07.981903Z","shell.execute_reply.started":"2022-07-15T03:06:07.862428Z","shell.execute_reply":"2022-07-15T03:06:07.980520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-Hot-Encoding\n\nlista = [0]*10\ny_train_encoding = []\nfor i in y_train:\n    lista[i] = 1\n    y_train_encoding.append(lista)\n    lista = 10*[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:07.983605Z","iopub.execute_input":"2022-07-15T03:06:07.983966Z","iopub.status.idle":"2022-07-15T03:06:08.181183Z","shell.execute_reply.started":"2022-07-15T03:06:07.983935Z","shell.execute_reply":"2022-07-15T03:06:08.180331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_encoding = np.array(y_train_encoding)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.182401Z","iopub.execute_input":"2022-07-15T03:06:08.183057Z","iopub.status.idle":"2022-07-15T03:06:08.238187Z","shell.execute_reply.started":"2022-07-15T03:06:08.183022Z","shell.execute_reply":"2022-07-15T03:06:08.237299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_encoding","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.240947Z","iopub.execute_input":"2022-07-15T03:06:08.241768Z","iopub.status.idle":"2022-07-15T03:06:08.250956Z","shell.execute_reply.started":"2022-07-15T03:06:08.241731Z","shell.execute_reply":"2022-07-15T03:06:08.249882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Building the network\n\nmodel = Sequential()\nmodel.add(Dense(350 , activation = 'relu' , input_shape = (784 , )))\nmodel.add(BatchNormalization())\nmodel.add(Dense(200 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(100 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(50 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(25 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(10 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(10 , activation = 'softmax'))\nmodel.compile(optimizer = 'adam' , loss = 'categorical_crossentropy' , metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.252502Z","iopub.execute_input":"2022-07-15T03:06:08.253034Z","iopub.status.idle":"2022-07-15T03:06:08.480116Z","shell.execute_reply.started":"2022-07-15T03:06:08.253002Z","shell.execute_reply":"2022-07-15T03:06:08.478965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a checkpoint to save the model weights that lead to the highest validation accuracy\n\ncheckpoint = ModelCheckpoint('weights.hdf5' , monitor = 'val_accuracy' , save_best_only = True )","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.481344Z","iopub.execute_input":"2022-07-15T03:06:08.482274Z","iopub.status.idle":"2022-07-15T03:06:08.486518Z","shell.execute_reply.started":"2022-07-15T03:06:08.482239Z","shell.execute_reply":"2022-07-15T03:06:08.485704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Forcing the model to stop when 5 epochs have passed without increasing the validation accuracy value\n\nearly_stopping = EarlyStopping(monitor = 'val_accuracy' , patience = 5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.487928Z","iopub.execute_input":"2022-07-15T03:06:08.488388Z","iopub.status.idle":"2022-07-15T03:06:08.496573Z","shell.execute_reply.started":"2022-07-15T03:06:08.488360Z","shell.execute_reply":"2022-07-15T03:06:08.495868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training the neural network\n\nhistory = model.fit(X_train , y_train_encoding, epochs = 40 , validation_split = 0.2 , callbacks = [checkpoint, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:06:08.497658Z","iopub.execute_input":"2022-07-15T03:06:08.498144Z","iopub.status.idle":"2022-07-15T03:08:33.131533Z","shell.execute_reply.started":"2022-07-15T03:06:08.498115Z","shell.execute_reply":"2022-07-15T03:08:33.130358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Higher validation accuracy value\n\nmax(history.history['val_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:08:33.134287Z","iopub.execute_input":"2022-07-15T03:08:33.134767Z","iopub.status.idle":"2022-07-15T03:08:33.141666Z","shell.execute_reply.started":"2022-07-15T03:08:33.134721Z","shell.execute_reply":"2022-07-15T03:08:33.140334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of epochs\n\nn_epochs = len(history.history['val_accuracy'])\n\nn_epochs","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:08:33.143990Z","iopub.execute_input":"2022-07-15T03:08:33.144629Z","iopub.status.idle":"2022-07-15T03:08:33.157665Z","shell.execute_reply.started":"2022-07-15T03:08:33.144577Z","shell.execute_reply":"2022-07-15T03:08:33.155562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the weights that generate the highest validation accuracy\n\nmodel.load_weights('weights.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:08:33.159495Z","iopub.execute_input":"2022-07-15T03:08:33.160013Z","iopub.status.idle":"2022-07-15T03:08:33.200409Z","shell.execute_reply.started":"2022-07-15T03:08:33.159958Z","shell.execute_reply":"2022-07-15T03:08:33.199113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(list(range(1 , n_epochs + 1, 1)) , history.history['accuracy'] , label = 'accuracy')\nplt.plot(list(range(1 , n_epochs + 1, 1)) , history.history['val_accuracy'] , label = 'val_accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:08:33.202096Z","iopub.execute_input":"2022-07-15T03:08:33.202686Z","iopub.status.idle":"2022-07-15T03:09:32.942979Z","shell.execute_reply.started":"2022-07-15T03:08:33.202637Z","shell.execute_reply":"2022-07-15T03:09:32.941175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.array(test)\npredictions = np.argmax(model.predict(X_test), axis=-1)\ntest['Label'] = predictions\ntest['ImageId'] = list(range(1, 28001 , 1))\ntest[['ImageId' , 'Label']].to_csv('submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T03:09:32.944958Z","iopub.execute_input":"2022-07-15T03:09:32.945475Z","iopub.status.idle":"2022-07-15T03:09:36.378153Z","shell.execute_reply.started":"2022-07-15T03:09:32.945423Z","shell.execute_reply":"2022-07-15T03:09:36.376600Z"},"trusted":true},"execution_count":null,"outputs":[]}]}