{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"904fd0017dd75d71251702c295c89dd1a660f1ce"},"cell_type":"code","source":"import math\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport time #to measure execution time","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Data loading"},{"metadata":{"trusted":true,"_uuid":"780d2274e6f4c0273519541e1365f51c9aecea9d"},"cell_type":"code","source":"#importing the data as pandas DataFrames\ntrain_data = pd.read_csv(\"../input/train.csv\")\ntest_data = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9993175f681a43e3a8ffa6d047f5763beb8f62a"},"cell_type":"code","source":"train_data.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2524d47896edb2bab99fe9dfc742b08f35a5bc0"},"cell_type":"code","source":"test_data.tail() #there is no label in the test dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26d6a78593b3513b545946296360627c7bce1141"},"cell_type":"code","source":"#now, let's convert the data to numpy arrays\n#we could use the pandas as_matrix() method, but we received a warning\n#FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n#x_test = test_data.as_matrix()\n#x_train = train_data.drop(['label'], axis=1).as_matrix()\n#y_train = train_data['label'].as_matrix()\n\nx_test = test_data.values\nx_train = train_data.drop(['label'], axis=1).values\ny_train = train_data['label'].values\n\nx_test.shape, x_train.shape, y_train.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a5e73262e934d7d648cb927637ea795c527f784"},"cell_type":"code","source":"# We split the known data into train and validation sets\n#let's choose 20% for validation set\nfrom sklearn.model_selection import train_test_split\nrandom_seed = 23\nx_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.14, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"98fd627b024339ed635a1fcfeb7b00717378c555"},"cell_type":"markdown","source":"To see which variables we have in memory, we can run *whos*"},{"metadata":{"trusted":true,"_uuid":"0d6b4efe0f1c4ceb0ab9b31220d4361247d6d198"},"cell_type":"code","source":"whos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87e9ea66c82ac56d26db6c0555fc9b3cf029deee"},"cell_type":"code","source":"#we scale the values using a linear interpolation, assuming that the values of the pixel intensities are initially from 0 to 255\nxmin = 0.\nxmax = 255. #the dot is to make them non-integer numbers, so that the result of the interpolation is also non-integer\nymin = 0\nymax = 1\n\nx_train = (x_train-xmin)*(ymax-ymin)/(xmax-xmin)\nx_test = (x_test-xmin)*(ymax-ymin)/(xmax-xmin)\n\nx_train.shape, x_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51396fa3e8f70a169c8b2fc7786526f683182dfb"},"cell_type":"code","source":"x_train[10] #notice the values are now scaled from 0 to 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6448d1094c4d93fda56f5c4b23745436b8311fd5"},"cell_type":"code","source":"plt.imshow(x_train[100].reshape([28,28]),'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77d8f149e92774fa6545a18aff18433c8edb915f"},"cell_type":"code","source":"x_train=x_train.reshape(x_train.shape[0], 28, 28,1)\nx_val=x_val.reshape(x_val.shape[0], 28, 28,1)\nx_test=x_test.reshape(x_test.shape[0], 28, 28,1)\n\nx_train.shape, x_val.shape, x_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85760f8e816e2fe57e5e9dddba40fc4fdd7f21a4"},"cell_type":"markdown","source":"We define a simple ANN model:"},{"metadata":{"trusted":true,"_uuid":"1bb5946f968e84a658dd5dd4fc1168c9debffef4"},"cell_type":"code","source":"from keras.models import Sequential #we import the class Sequential: Sequential Layer ANN Models\nfrom keras.layers import Dense, BatchNormalization, Dropout\nfrom keras.optimizers import SGD #there are more optimizers available too\nfrom keras.regularizers import l2 #we use weight regularization\nfrom keras.layers import Conv2D, MaxPool2D, Flatten\n\nlr = 0.010#0.01\nbs = 256 #batch size\nnb = math.ceil(len(x_train)/bs) # batch number\n\nmodel = Sequential([\n    Conv2D(128, 3, activation='relu', padding='same', input_shape=(28,28,1)),\n    MaxPool2D(),\n    Flatten(),\n    Dense(512, activation='relu', input_shape=(784,)),#we could have not added the input_shape information. it could have been inferred by keras\n    #BatchNormalization(),#\n    Dropout(0.25),\n    Dense(10, activation='softmax', kernel_regularizer=l2(0.002))\n          #because it's a classifier, we use softmax. 10 neurons at the end, because there are 10 classes (digits) to classify\n])\n\n#categorical_crossentropy: needs that we explicitly do the one hot encoding. if it's sparse, it does it internally\n#we can get several metrics during the model training\n#we use SGD with Nesterov momentum update\nmodel.compile(SGD(lr, momentum=0.9, nesterov=True), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n\nmodel.summary() #shows us the layer details","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"371aff91ff29417674262f58f3ec14e5d9cefbec"},"cell_type":"code","source":"log = model.fit(x_train, y_train, batch_size=bs, epochs=23, validation_data=[x_val, y_val]) #model training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1651e43bb024839033b55e8895b3cad41cb8fe0b"},"cell_type":"code","source":"def show_results(model, log, cycling=False):\n    loss, acc = model.evaluate(x_val, y_val, batch_size=512, verbose=False)\n    print(f'Loss     = {loss:.4f}')\n    print(f'Accuracy = {acc:.4f}')\n    \n    val_loss = log.history['val_loss']\n    val_acc = log.history['val_acc']\n    if cycling:\n        val_loss += [loss]\n        val_acc += [acc]\n        \n    fig, axes = plt.subplots(1, 2, figsize=(14,4))\n    ax1, ax2 = axes\n    ax1.plot(log.history['loss'], label='train')\n    ax1.plot(val_loss, label='test')\n    ax1.set_xlabel('epoch'); ax1.set_ylabel('loss')\n    ax2.plot(log.history['acc'], label='train')\n    ax2.plot(val_acc, label='test')\n    ax2.set_xlabel('epoch'); ax2.set_ylabel('accuracy')\n    for ax in axes: ax.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acd9b0ec05f0b66b543aede72f4cfb196f275eea"},"cell_type":"code","source":"show_results(model, log)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd07d2b192626f2815f1efb43b254d12f26ad977"},"cell_type":"code","source":"# predict results\ny_test = model.predict(x_test)\n\n# select the indix with the maximum probability\ny_test = np.argmax(y_test,axis = 1)\n\ny_test = pd.Series(y_test,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a7f80774d24b2a724380bac7c290993f32683a3"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),y_test],axis = 1)\n\nsubmission.to_csv(\"predictions.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}