{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-success\">\n    <h1 align=\"center\">Digit Recognizer Competition</h1>\n    <h3 align=\"center\">Deep Learning Tutorial</h3>\n    <h4 align=\"center\"><a href=\"http://www.iran-machinelearning.ir\">Soheil Tehranipour</a></h5>\n</div>","metadata":{}},{"cell_type":"markdown","source":"<center><img src=\"https://media-exp2.licdn.com/dms/image/C4D12AQHAia5q8RDExg/article-cover_image-shrink_600_2000/0/1600202596769?e=1662595200&v=beta&t=cry4Vbh-xq8dPOSMladMps-5W0yApNHrpMioTyJzsJs\" width=70%></center>","metadata":{}},{"cell_type":"code","source":"# Some basics importations\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom keras.layers import Dense , BatchNormalization\nfrom keras.models import Sequential\nfrom keras.callbacks import ModelCheckpoint , EarlyStopping\nfrom IPython.display import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T11:13:50.812052Z","iopub.execute_input":"2022-07-07T11:13:50.812680Z","iopub.status.idle":"2022-07-07T11:14:01.871502Z","shell.execute_reply.started":"2022-07-07T11:13:50.812588Z","shell.execute_reply":"2022-07-07T11:14:01.869949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:01.874181Z","iopub.execute_input":"2022-07-07T11:14:01.874817Z","iopub.status.idle":"2022-07-07T11:14:07.725489Z","shell.execute_reply.started":"2022-07-07T11:14:01.874782Z","shell.execute_reply":"2022-07-07T11:14:07.724212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:07.727316Z","iopub.execute_input":"2022-07-07T11:14:07.727743Z","iopub.status.idle":"2022-07-07T11:14:07.738735Z","shell.execute_reply.started":"2022-07-07T11:14:07.727701Z","shell.execute_reply":"2022-07-07T11:14:07.737248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observing some training data\n\ntrain.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:07.740599Z","iopub.execute_input":"2022-07-07T11:14:07.740959Z","iopub.status.idle":"2022-07-07T11:14:07.770679Z","shell.execute_reply.started":"2022-07-07T11:14:07.740927Z","shell.execute_reply":"2022-07-07T11:14:07.769431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for missing values\n\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:07.773521Z","iopub.execute_input":"2022-07-07T11:14:07.774842Z","iopub.status.idle":"2022-07-07T11:14:07.828336Z","shell.execute_reply.started":"2022-07-07T11:14:07.774800Z","shell.execute_reply":"2022-07-07T11:14:07.827161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Total missing values\n\nsum(train.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:07.829898Z","iopub.execute_input":"2022-07-07T11:14:07.830332Z","iopub.status.idle":"2022-07-07T11:14:07.882258Z","shell.execute_reply.started":"2022-07-07T11:14:07.830295Z","shell.execute_reply":"2022-07-07T11:14:07.881434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = np.array(train['label'])\nX_train = train.drop('label' , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:07.884303Z","iopub.execute_input":"2022-07-07T11:14:07.884745Z","iopub.status.idle":"2022-07-07T11:14:08.007261Z","shell.execute_reply.started":"2022-07-07T11:14:07.884704Z","shell.execute_reply":"2022-07-07T11:14:08.006143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-Hot-Encoding\n\nlista = [0]*10\ny_train_encoding = []\nfor i in y_train:\n    lista[i] = 1\n    y_train_encoding.append(lista)\n    lista = 10*[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.013323Z","iopub.execute_input":"2022-07-07T11:14:08.013683Z","iopub.status.idle":"2022-07-07T11:14:08.227030Z","shell.execute_reply.started":"2022-07-07T11:14:08.013654Z","shell.execute_reply":"2022-07-07T11:14:08.225881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_encoding = np.array(y_train_encoding)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.228349Z","iopub.execute_input":"2022-07-07T11:14:08.229153Z","iopub.status.idle":"2022-07-07T11:14:08.327925Z","shell.execute_reply.started":"2022-07-07T11:14:08.229105Z","shell.execute_reply":"2022-07-07T11:14:08.326537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_encoding","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.329918Z","iopub.execute_input":"2022-07-07T11:14:08.330416Z","iopub.status.idle":"2022-07-07T11:14:08.340905Z","shell.execute_reply.started":"2022-07-07T11:14:08.330351Z","shell.execute_reply":"2022-07-07T11:14:08.339802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Building the network\n\nmodel = Sequential()\nmodel.add(Dense(350 , activation = 'relu' , input_shape = (784 , )))\nmodel.add(BatchNormalization())\nmodel.add(Dense(200 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(100 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(50 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(25 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(10 , activation = 'relu' ))\nmodel.add(BatchNormalization())\nmodel.add(Dense(10 , activation = 'softmax'))\nmodel.compile(optimizer = 'adam' , loss = 'categorical_crossentropy' , metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.342435Z","iopub.execute_input":"2022-07-07T11:14:08.342900Z","iopub.status.idle":"2022-07-07T11:14:08.595265Z","shell.execute_reply.started":"2022-07-07T11:14:08.342854Z","shell.execute_reply":"2022-07-07T11:14:08.592991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a checkpoint to save the model weights that lead to the highest validation accuracy\n\ncheckpoint = ModelCheckpoint('weights.hdf5' , monitor = 'val_accuracy' , save_best_only = True )","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.596661Z","iopub.execute_input":"2022-07-07T11:14:08.597410Z","iopub.status.idle":"2022-07-07T11:14:08.603542Z","shell.execute_reply.started":"2022-07-07T11:14:08.597346Z","shell.execute_reply":"2022-07-07T11:14:08.602424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Forcing the model to stop when 5 epochs have passed without increasing the validation accuracy value\n\nearly_stopping = EarlyStopping(monitor = 'val_accuracy' , patience = 5)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.605792Z","iopub.execute_input":"2022-07-07T11:14:08.607048Z","iopub.status.idle":"2022-07-07T11:14:08.616877Z","shell.execute_reply.started":"2022-07-07T11:14:08.607008Z","shell.execute_reply":"2022-07-07T11:14:08.615671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training the neural network\n\nhistory = model.fit(X_train , y_train_encoding, epochs = 40 , validation_split = 0.2 , callbacks = [checkpoint, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:14:08.618514Z","iopub.execute_input":"2022-07-07T11:14:08.619610Z","iopub.status.idle":"2022-07-07T11:18:21.701510Z","shell.execute_reply.started":"2022-07-07T11:14:08.619561Z","shell.execute_reply":"2022-07-07T11:18:21.700446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Higher validation accuracy value\n\nmax(history.history['val_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:18:21.703753Z","iopub.execute_input":"2022-07-07T11:18:21.704139Z","iopub.status.idle":"2022-07-07T11:18:21.710995Z","shell.execute_reply.started":"2022-07-07T11:18:21.704097Z","shell.execute_reply":"2022-07-07T11:18:21.709928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of epochs\n\nn_epochs = len(history.history['val_accuracy'])\n\nn_epochs","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:18:21.712485Z","iopub.execute_input":"2022-07-07T11:18:21.713110Z","iopub.status.idle":"2022-07-07T11:18:21.724519Z","shell.execute_reply.started":"2022-07-07T11:18:21.713075Z","shell.execute_reply":"2022-07-07T11:18:21.723550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the weights that generate the highest validation accuracy\n\nmodel.load_weights('weights.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:18:21.725981Z","iopub.execute_input":"2022-07-07T11:18:21.726363Z","iopub.status.idle":"2022-07-07T11:18:21.762422Z","shell.execute_reply.started":"2022-07-07T11:18:21.726329Z","shell.execute_reply":"2022-07-07T11:18:21.761251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(list(range(1 , n_epochs + 1, 1)) , history.history['accuracy'] , label = 'accuracy')\nplt.plot(list(range(1 , n_epochs + 1, 1)) , history.history['val_accuracy'] , label = 'val_accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:18:21.764266Z","iopub.execute_input":"2022-07-07T11:18:21.765117Z","iopub.status.idle":"2022-07-07T11:18:22.025362Z","shell.execute_reply.started":"2022-07-07T11:18:21.765064Z","shell.execute_reply":"2022-07-07T11:18:22.024142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.array(test)\npredictions = np.argmax(model.predict(X_test), axis=-1)\ntest['Label'] = predictions\ntest['ImageId'] = list(range(1, 28001 , 1))\ntest[['ImageId' , 'Label']].to_csv('submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:18:22.027682Z","iopub.execute_input":"2022-07-07T11:18:22.028516Z","iopub.status.idle":"2022-07-07T11:18:25.228739Z","shell.execute_reply.started":"2022-07-07T11:18:22.028463Z","shell.execute_reply":"2022-07-07T11:18:25.227558Z"},"trusted":true},"execution_count":null,"outputs":[]}]}