{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.preprocessing import binarize\nfrom sklearn.model_selection import GridSearchCV\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\ntraining_df = pd.read_csv('../input/train.csv')\n#Image Size = 28*28\nX = training_df.drop('label', axis=1)\ny = training_df['label']\n\nX.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0e052858343528cef4d04de60ec7bd6af09bd2b"},"cell_type":"markdown","source":"# Normalization\nPixel values are between [0, 255] which are scaled to [0, 1]\nStandard Scaling with unit variance is not as much effective as scaling the pixel values for the reason that most of the pixels are 0 in space."},{"metadata":{"trusted":true,"_uuid":"2ee3bf2d6b4db0f46c1d54780a228c20e913fc96"},"cell_type":"code","source":"X_scaled = X / 255.0\nX_scaled.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c1407d05c7d8b4f7a862d9792cfd98e0c071f9b"},"cell_type":"markdown","source":"# Generate Test and Train Data"},{"metadata":{"trusted":true,"_uuid":"cb95bc1d3a3c048f0222343e5f9e39b27c5e09c2"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_X, test_X, train_y, test_y = train_test_split(X_scaled, y, test_size=0.3, random_state=1, stratify=y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6236592b562428869582f921883a282daf0f1d65"},"cell_type":"markdown","source":"# Train using MLPClassifier"},{"metadata":{"trusted":true,"_uuid":"baf0183887c6405f81f6a63cf9e0ca33d9c69732"},"cell_type":"code","source":"model = MLPClassifier(solver='lbfgs', activation='relu', learning_rate_init = 0.01, max_iter=400, alpha=1e-3, hidden_layer_sizes=(64,28), random_state=1)\nmodel.fit(train_X, train_y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be13dc62ae1a3b20a2f534a0794f9b9b08da096a"},"cell_type":"markdown","source":"# Label Classification Distribution"},{"metadata":{"trusted":true,"_uuid":"3a619133c3593950da1080df4756edf06b2d43b3"},"cell_type":"code","source":"y.value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f00102ec0ff34cd8345b3cd76ae28a2cbcbe0cf"},"cell_type":"markdown","source":"# Accuracy Score\nSince, label classifications are almost uniform, normal accuracy score will be enough to find the accuracy."},{"metadata":{"trusted":true,"_uuid":"4d82a888df91c32feb716d4899e437e870d2bdcb"},"cell_type":"code","source":"train_y_pred = model.predict(train_X)\ntest_y_pred = model.predict(test_X)\n\n#Training Prediction Accuracy\nprint(accuracy_score(train_y.values, train_y_pred))\n#Test Prediction Accuracy\nprint(accuracy_score(test_y.values, test_y_pred))\n\n#Classification Report\nprint(classification_report(test_y.values, test_y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"062fc87be5b2338ebe1f989232061555211b03e7"},"cell_type":"markdown","source":"# HyperTuning Parameters"},{"metadata":{"trusted":true,"_uuid":"c36a2607524873ffa1e4d0a15ac329d519b1bff2"},"cell_type":"code","source":"#from sklearn.model_selection import GridSearchCV\n\n#parameter_options = {\n#    \"learning_rate_init\" : (0.1, 0.01, 0.005, 0.001),\n#    \"alpha\" : (1e-2, 1e-3, 1e-4, 1e-5)\n#}\n\n#tuning_model = MLPClassifier(solver='lbfgs', activation='relu', max_iter=400, hidden_layer_sizes=(64,28), random_state=1)\n\n#clf = GridSearchCV(tuning_model, parameter_options, cv=5)\n#clf.fit(X_scaled, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2006104bf0a20cdf52e6bd05a5ebc36b47e288e4"},"cell_type":"code","source":"hyper_parameter_json = '{\"mean_fit_time\":{\"0\":62.2645419121,\"1\":61.8836660862,\"2\":61.3482870579,\"3\":61.9847536564,\"4\":64.099307251,\"5\":64.139607954,\"6\":64.1262664795,\"7\":64.1702826977,\"8\":62.9692382336,\"9\":62.8342501163,\"10\":62.9287622452,\"11\":62.7448991299,\"12\":63.1923060417,\"13\":63.5381466866,\"14\":63.3286220551,\"15\":62.867953968},\"std_fit_time\":{\"0\":1.2380253246,\"1\":0.7392680755,\"2\":0.8371015025,\"3\":1.2630434989,\"4\":3.1459295475,\"5\":3.4569018506,\"6\":3.3554040279,\"7\":3.3614496645,\"8\":3.7456630793,\"9\":3.5959209416,\"10\":3.6718319373,\"11\":3.6167472564,\"12\":3.345812199,\"13\":3.5025556031,\"14\":3.47975575,\"15\":3.2289069236},\"mean_score_time\":{\"0\":0.0544639587,\"1\":0.0534081936,\"2\":0.0537557602,\"3\":0.0544393063,\"4\":0.0532652855,\"5\":0.0528447628,\"6\":0.0531326294,\"7\":0.0537816048,\"8\":0.0536803246,\"9\":0.0548650265,\"10\":0.0528648376,\"11\":0.0532189846,\"12\":0.0533833981,\"13\":0.0529069901,\"14\":0.0531209946,\"15\":0.0529269218},\"std_score_time\":{\"0\":0.0024103018,\"1\":0.0027094884,\"2\":0.0028988523,\"3\":0.0025701715,\"4\":0.0004388501,\"5\":0.0005909962,\"6\":0.0008000669,\"7\":0.0013918448,\"8\":0.0017466922,\"9\":0.0016980453,\"10\":0.0007532973,\"11\":0.0010817515,\"12\":0.0016575218,\"13\":0.0006490277,\"14\":0.0010582231,\"15\":0.0004445466},\"param_alpha\":{\"0\":0.01,\"1\":0.01,\"2\":0.01,\"3\":0.01,\"4\":0.001,\"5\":0.001,\"6\":0.001,\"7\":0.001,\"8\":0.0001,\"9\":0.0001,\"10\":0.0001,\"11\":0.0001,\"12\":0.00001,\"13\":0.00001,\"14\":0.00001,\"15\":0.00001},\"param_learning_rate_init\":{\"0\":0.1,\"1\":0.01,\"2\":0.005,\"3\":0.001,\"4\":0.1,\"5\":0.01,\"6\":0.005,\"7\":0.001,\"8\":0.1,\"9\":0.01,\"10\":0.005,\"11\":0.001,\"12\":0.1,\"13\":0.01,\"14\":0.005,\"15\":0.001},\"params\":{\"0\":{\"alpha\":0.01,\"learning_rate_init\":0.1},\"1\":{\"alpha\":0.01,\"learning_rate_init\":0.01},\"2\":{\"alpha\":0.01,\"learning_rate_init\":0.005},\"3\":{\"alpha\":0.01,\"learning_rate_init\":0.001},\"4\":{\"alpha\":0.001,\"learning_rate_init\":0.1},\"5\":{\"alpha\":0.001,\"learning_rate_init\":0.01},\"6\":{\"alpha\":0.001,\"learning_rate_init\":0.005},\"7\":{\"alpha\":0.001,\"learning_rate_init\":0.001},\"8\":{\"alpha\":0.0001,\"learning_rate_init\":0.1},\"9\":{\"alpha\":0.0001,\"learning_rate_init\":0.01},\"10\":{\"alpha\":0.0001,\"learning_rate_init\":0.005},\"11\":{\"alpha\":0.0001,\"learning_rate_init\":0.001},\"12\":{\"alpha\":0.00001,\"learning_rate_init\":0.1},\"13\":{\"alpha\":0.00001,\"learning_rate_init\":0.01},\"14\":{\"alpha\":0.00001,\"learning_rate_init\":0.005},\"15\":{\"alpha\":0.00001,\"learning_rate_init\":0.001}},\"split0_test_score\":{\"0\":0.9650208209,\"1\":0.9650208209,\"2\":0.9650208209,\"3\":0.9650208209,\"4\":0.9654967281,\"5\":0.9654967281,\"6\":0.9654967281,\"7\":0.9654967281,\"8\":0.9653777513,\"9\":0.9653777513,\"10\":0.9653777513,\"11\":0.9653777513,\"12\":0.9657346817,\"13\":0.9657346817,\"14\":0.9657346817,\"15\":0.9657346817},\"split1_test_score\":{\"0\":0.9656075211,\"1\":0.9656075211,\"2\":0.9656075211,\"3\":0.9656075211,\"4\":0.9659645365,\"5\":0.9659645365,\"6\":0.9659645365,\"7\":0.9659645365,\"8\":0.9670355825,\"9\":0.9670355825,\"10\":0.9670355825,\"11\":0.9670355825,\"12\":0.9659645365,\"13\":0.9659645365,\"14\":0.9659645365,\"15\":0.9659645365},\"split2_test_score\":{\"0\":0.9645195857,\"1\":0.9645195857,\"2\":0.9645195857,\"3\":0.9645195857,\"4\":0.9651148946,\"5\":0.9651148946,\"6\":0.9651148946,\"7\":0.9651148946,\"8\":0.9651148946,\"9\":0.9651148946,\"10\":0.9651148946,\"11\":0.9651148946,\"12\":0.9649958328,\"13\":0.9649958328,\"14\":0.9649958328,\"15\":0.9649958328},\"split3_test_score\":{\"0\":0.9623675122,\"1\":0.9623675122,\"2\":0.9623675122,\"3\":0.9623675122,\"4\":0.9632011433,\"5\":0.9632011433,\"6\":0.9632011433,\"7\":0.9632011433,\"8\":0.9617720615,\"9\":0.9617720615,\"10\":0.9617720615,\"11\":0.9617720615,\"12\":0.9620102418,\"13\":0.9620102418,\"14\":0.9620102418,\"15\":0.9620102418},\"split4_test_score\":{\"0\":0.9667698904,\"1\":0.9667698904,\"2\":0.9667698904,\"3\":0.9667698904,\"4\":0.9664125774,\"5\":0.9664125774,\"6\":0.9664125774,\"7\":0.9664125774,\"8\":0.9677227251,\"9\":0.9677227251,\"10\":0.9677227251,\"11\":0.9677227251,\"12\":0.9665316818,\"13\":0.9665316818,\"14\":0.9665316818,\"15\":0.9665316818},\"mean_test_score\":{\"0\":0.9648571429,\"1\":0.9648571429,\"2\":0.9648571429,\"3\":0.9648571429,\"4\":0.9652380952,\"5\":0.9652380952,\"6\":0.9652380952,\"7\":0.9652380952,\"8\":0.9654047619,\"9\":0.9654047619,\"10\":0.9654047619,\"11\":0.9654047619,\"12\":0.965047619,\"13\":0.965047619,\"14\":0.965047619,\"15\":0.965047619},\"std_test_score\":{\"0\":0.0014530598,\"1\":0.0014530598,\"2\":0.0014530598,\"3\":0.0014530598,\"4\":0.0011078317,\"5\":0.0011078317,\"6\":0.0011078317,\"7\":0.0011078317,\"8\":0.0020643383,\"9\":0.0020643383,\"10\":0.0020643383,\"11\":0.0020643383,\"12\":0.001596234,\"13\":0.001596234,\"14\":0.001596234,\"15\":0.001596234},\"rank_test_score\":{\"0\":13,\"1\":13,\"2\":13,\"3\":13,\"4\":5,\"5\":5,\"6\":5,\"7\":5,\"8\":1,\"9\":1,\"10\":1,\"11\":1,\"12\":9,\"13\":9,\"14\":9,\"15\":9},\"split0_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"split1_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"split2_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"split3_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"split4_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"mean_train_score\":{\"0\":1.0,\"1\":1.0,\"2\":1.0,\"3\":1.0,\"4\":1.0,\"5\":1.0,\"6\":1.0,\"7\":1.0,\"8\":1.0,\"9\":1.0,\"10\":1.0,\"11\":1.0,\"12\":1.0,\"13\":1.0,\"14\":1.0,\"15\":1.0},\"std_train_score\":{\"0\":0.0,\"1\":0.0,\"2\":0.0,\"3\":0.0,\"4\":0.0,\"5\":0.0,\"6\":0.0,\"7\":0.0,\"8\":0.0,\"9\":0.0,\"10\":0.0,\"11\":0.0,\"12\":0.0,\"13\":0.0,\"14\":0.0,\"15\":0.0}}'\nhyper_parameter_df = pd.read_json(hyper_parameter_json)\nhyper_parameter_df.sort_values('rank_test_score', inplace=True)\nhyper_parameter_df[[\"param_alpha\", \"param_learning_rate_init\", \"mean_test_score\", \"std_test_score\", \"rank_test_score\"]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17e1b0d2d74d9deb29fcdcad180f188ff18ab58b"},"cell_type":"markdown","source":"# Selecting Params\n**alpha** = 0.00100,\n**learning_init_rate** = 0.1"},{"metadata":{"_uuid":"7d799fff8633da4fd489d9e1722da75e83fe5347"},"cell_type":"markdown","source":"# Final Model Creation"},{"metadata":{"trusted":true,"_uuid":"c258178210e40541d6d286da995ed961bb743bd5"},"cell_type":"code","source":"nn_model = MLPClassifier(solver='lbfgs', activation='relu', learning_rate_init = 0.1, max_iter=400, alpha=0.001, hidden_layer_sizes=(64,28), random_state=1)\n#Fitting on Complete Scaled Data\nnn_model.fit(X_scaled, y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e31163971069e3ea22fa375827dbb2ced56d637c"},"cell_type":"markdown","source":"# Predict on Test Data"},{"metadata":{"trusted":true,"_uuid":"28c09e2e466af7389c18238e01101558a0384d59"},"cell_type":"code","source":"# path to file you will use for predictions\ntest_data_path = '../input/test.csv'\n\n# read test data file using pandas\ntest_data = pd.read_csv(test_data_path)\n#Scaling Test Data\ntest_scaled_data = test_data / 255.0\n#Predicting Test Data\ntest_predictions = nn_model.predict(test_scaled_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"195e6a04eb22e88913c8057b567545a7330c59c5"},"cell_type":"markdown","source":"# Output as csv\n"},{"metadata":{"trusted":true,"_uuid":"960abfd3c8a9550561c9679d0c4b867b76281017"},"cell_type":"code","source":"result = pd.DataFrame(test_predictions, columns=['Label'])\nresult.reset_index(inplace=True)\nresult.rename(columns={'index': 'ImageId'}, inplace=True)\nresult['ImageId'] = result['ImageId']+1\nresult.to_csv('output.csv', index=False)\nresult.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea9c21083f861273196fc88ceef64f091b0231c4"},"cell_type":"markdown","source":"# Remaining\nDecreasing the train data and testing the accuracy on test data."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}