{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MNIST Ensemble Classification","metadata":{}},{"cell_type":"markdown","source":"My goal for this notebook is \n1. To train a number of classifiers on the MNIST dataset\n2. To ensemble the classifiers together to produce a stronger classifier","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:35:27.602939Z","iopub.execute_input":"2022-08-06T13:35:27.604440Z","iopub.status.idle":"2022-08-06T13:35:35.206341Z","shell.execute_reply.started":"2022-08-06T13:35:27.604269Z","shell.execute_reply":"2022-08-06T13:35:35.204998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import the classifiers we're going to use:","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.neural_network import MLPClassifier\n\n\nfrom sklearn.ensemble import VotingClassifier\n\nfrom sklearn.metrics import accuracy_score\n","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:35:35.208701Z","iopub.execute_input":"2022-08-06T13:35:35.209403Z","iopub.status.idle":"2022-08-06T13:35:35.776094Z","shell.execute_reply.started":"2022-08-06T13:35:35.209362Z","shell.execute_reply":"2022-08-06T13:35:35.774889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import the data for our training sets:","metadata":{}},{"cell_type":"code","source":"X_train_full = pd.read_csv('../input/digit-recognizer/train.csv')\ny_train_full = X_train_full['label']\nX_train_full.drop('label', axis=1, inplace=True)\nX_test = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:35:35.777419Z","iopub.execute_input":"2022-08-06T13:35:35.777767Z","iopub.status.idle":"2022-08-06T13:35:41.200056Z","shell.execute_reply.started":"2022-08-06T13:35:35.777736Z","shell.execute_reply":"2022-08-06T13:35:41.198945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Take a validation set:","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full, test_size=7000, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:35:41.202530Z","iopub.execute_input":"2022-08-06T13:35:41.202972Z","iopub.status.idle":"2022-08-06T13:35:41.488390Z","shell.execute_reply.started":"2022-08-06T13:35:41.202933Z","shell.execute_reply":"2022-08-06T13:35:41.487196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's begin training the first of our classifiers:","metadata":{}},{"cell_type":"markdown","source":"## Random Forest Classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparam_grid = [\n    {'n_estimators': [500],\n     'max_depth': [2,3,5],\n     'min_samples_split': [2, 5],\n     'min_samples_leaf': [3, 4, 5]}\n]\n\nrnd_clf = RandomForestClassifier(random_state=42)\n\ngrid_search_forest = GridSearchCV(estimator = rnd_clf, param_grid = param_grid, \n                          cv = 3, n_jobs = -1, verbose = 2)\n\ngrid_search_forest.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:35:41.489862Z","iopub.execute_input":"2022-08-06T13:35:41.490236Z","iopub.status.idle":"2022-08-06T13:42:25.374375Z","shell.execute_reply.started":"2022-08-06T13:35:41.490203Z","shell.execute_reply":"2022-08-06T13:42:25.373144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_forest.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:42:25.376251Z","iopub.execute_input":"2022-08-06T13:42:25.376942Z","iopub.status.idle":"2022-08-06T13:42:25.385669Z","shell.execute_reply.started":"2022-08-06T13:42:25.376900Z","shell.execute_reply":"2022-08-06T13:42:25.384388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Going to perform one more gridsearch to further fine tune the params","metadata":{}},{"cell_type":"code","source":"param_grid = [\n    {'n_estimators': [500],\n     'max_depth': [None,5],\n     'min_samples_split': [1, 2],\n     'min_samples_leaf': [1, 3]}\n]\n\nrnd_clf = RandomForestClassifier(random_state=42)\n\ngrid_search_forest = GridSearchCV(estimator = rnd_clf, param_grid = param_grid, \n                          cv = 3, n_jobs = -1, verbose = 2)\n\ngrid_search_forest.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:42:25.387578Z","iopub.execute_input":"2022-08-06T13:42:25.388873Z","iopub.status.idle":"2022-08-06T13:48:34.509500Z","shell.execute_reply.started":"2022-08-06T13:42:25.388823Z","shell.execute_reply":"2022-08-06T13:48:34.508211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_forest.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:48:34.511338Z","iopub.execute_input":"2022-08-06T13:48:34.512573Z","iopub.status.idle":"2022-08-06T13:48:34.520598Z","shell.execute_reply.started":"2022-08-06T13:48:34.512524Z","shell.execute_reply":"2022-08-06T13:48:34.519383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ny_pred_forest = grid_search_forest.predict(X_valid)\n\nprint(rnd_clf.__class__.__name__, accuracy_score(y_valid, y_pred_forest))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:48:34.521948Z","iopub.execute_input":"2022-08-06T13:48:34.523219Z","iopub.status.idle":"2022-08-06T13:48:36.123451Z","shell.execute_reply.started":"2022-08-06T13:48:34.523159Z","shell.execute_reply":"2022-08-06T13:48:36.122042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Random Forest Classifier: 96.47%**","metadata":{}},{"cell_type":"markdown","source":"# Extra Trees Classifier","metadata":{}},{"cell_type":"code","source":"param_grid = [\n    {'n_estimators': [100],\n     'max_depth': [None,5],\n     'min_samples_split': [2, 3],\n     'min_samples_leaf': [1, 2]}\n]\n\next_clf = ExtraTreesClassifier(random_state=42)\n\ngrid_search_extra = GridSearchCV(estimator = ext_clf, param_grid = param_grid, \n                          cv = 3, n_jobs = -1, verbose = 2)\n\ngrid_search_extra.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:48:36.128902Z","iopub.execute_input":"2022-08-06T13:48:36.129709Z","iopub.status.idle":"2022-08-06T13:50:28.523283Z","shell.execute_reply.started":"2022-08-06T13:48:36.129660Z","shell.execute_reply":"2022-08-06T13:50:28.522325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_extra.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:50:28.524935Z","iopub.execute_input":"2022-08-06T13:50:28.525523Z","iopub.status.idle":"2022-08-06T13:50:28.532506Z","shell.execute_reply.started":"2022-08-06T13:50:28.525483Z","shell.execute_reply":"2022-08-06T13:50:28.531382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_extra = grid_search_extra.predict(X_valid)\n\nprint(rnd_clf.__class__.__name__, accuracy_score(y_valid, y_pred_extra))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:50:28.533873Z","iopub.execute_input":"2022-08-06T13:50:28.534233Z","iopub.status.idle":"2022-08-06T13:50:28.933556Z","shell.execute_reply.started":"2022-08-06T13:50:28.534201Z","shell.execute_reply":"2022-08-06T13:50:28.932378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Extra Trees Classifier: 96.59%**","metadata":{}},{"cell_type":"markdown","source":"# Linear SVC Classifier","metadata":{}},{"cell_type":"code","source":"svc_clf = LinearSVC(max_iter=1000, random_state=42)\n\nparam_grid = [\n    {'C': [0.01, 0.1, 1, 10]}\n]\n\ngrid_search_svc = GridSearchCV(estimator = svc_clf, param_grid = param_grid, \n                          cv = 3, n_jobs = -1, verbose = 2)\n\ngrid_search_svc.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:50:28.935017Z","iopub.execute_input":"2022-08-06T13:50:28.936099Z","iopub.status.idle":"2022-08-06T13:56:57.984261Z","shell.execute_reply.started":"2022-08-06T13:50:28.936063Z","shell.execute_reply":"2022-08-06T13:56:57.983010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_svc.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:56:57.986242Z","iopub.execute_input":"2022-08-06T13:56:57.986603Z","iopub.status.idle":"2022-08-06T13:56:57.994684Z","shell.execute_reply.started":"2022-08-06T13:56:57.986572Z","shell.execute_reply":"2022-08-06T13:56:57.993199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try another one with higher C params","metadata":{}},{"cell_type":"code","source":"param_grid = [\n    {'C': [5, 10, 20, 100]}\n]\n\ngrid_search_svc = GridSearchCV(estimator = svc_clf, param_grid = param_grid, \n                          cv = 3, n_jobs = -1, verbose = 2)\n\ngrid_search_svc.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T13:56:57.996346Z","iopub.execute_input":"2022-08-06T13:56:57.996723Z","iopub.status.idle":"2022-08-06T14:03:33.113269Z","shell.execute_reply.started":"2022-08-06T13:56:57.996681Z","shell.execute_reply":"2022-08-06T14:03:33.111657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_svc.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-08-06T14:03:33.114777Z","iopub.execute_input":"2022-08-06T14:03:33.115127Z","iopub.status.idle":"2022-08-06T14:03:33.123473Z","shell.execute_reply.started":"2022-08-06T14:03:33.115095Z","shell.execute_reply":"2022-08-06T14:03:33.122219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_svc = grid_search_svc.predict(X_valid)\n\nprint(svc_clf.__class__.__name__, accuracy_score(y_valid, y_pred_svc))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T14:03:33.125389Z","iopub.execute_input":"2022-08-06T14:03:33.125808Z","iopub.status.idle":"2022-08-06T14:03:33.229972Z","shell.execute_reply.started":"2022-08-06T14:03:33.125772Z","shell.execute_reply":"2022-08-06T14:03:33.228403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Linear SVC: 84.87%**","metadata":{}},{"cell_type":"markdown","source":"# Multi Layer Perceptrons","metadata":{}},{"cell_type":"code","source":"mlp_clf = MLPClassifier(hidden_layer_sizes=(10000,500), batch_size=512, verbose=1)\n\nmlp_clf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T14:03:33.231998Z","iopub.execute_input":"2022-08-06T14:03:33.233370Z","iopub.status.idle":"2022-08-06T14:28:27.602741Z","shell.execute_reply.started":"2022-08-06T14:03:33.233282Z","shell.execute_reply":"2022-08-06T14:28:27.601541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_mlp = mlp_clf.predict(X_valid)\n\nprint(mlp_clf.__class__.__name__, accuracy_score(y_valid, y_pred_mlp))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T14:28:27.604437Z","iopub.execute_input":"2022-08-06T14:28:27.605046Z","iopub.status.idle":"2022-08-06T14:28:30.906034Z","shell.execute_reply.started":"2022-08-06T14:28:27.605008Z","shell.execute_reply":"2022-08-06T14:28:30.901707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MLP Classifier Accuracy: 96.84%**","metadata":{}},{"cell_type":"markdown","source":"# Voting Classifier","metadata":{}},{"cell_type":"markdown","source":"Note that I am removing the LinearSVC classifier from voting because the poor accuracy may detract from the ensembles score","metadata":{}},{"cell_type":"code","source":"voting_clf = VotingClassifier(\n    estimators=[('mlp', mlp_clf), ('rf', grid_search_forest), ('ex',grid_search_extra)],\n    voting='soft'\n)\n\nvoting_clf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T14:28:30.911736Z","iopub.execute_input":"2022-08-06T14:28:30.917068Z","iopub.status.idle":"2022-08-06T15:11:05.997689Z","shell.execute_reply.started":"2022-08-06T14:28:30.916988Z","shell.execute_reply":"2022-08-06T15:11:05.996383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = voting_clf.predict(X_valid)\n\nprint(voting_clf.__class__.__name__, accuracy_score(y_valid, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:11:05.999606Z","iopub.execute_input":"2022-08-06T15:11:06.000213Z","iopub.status.idle":"2022-08-06T15:11:11.238951Z","shell.execute_reply.started":"2022-08-06T15:11:06.000170Z","shell.execute_reply":"2022-08-06T15:11:11.237636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Voting Classifier Accuracy: 97.83%**","metadata":{}},{"cell_type":"markdown","source":"# Final Ensemble Submission","metadata":{}},{"cell_type":"code","source":"final_pred = voting_clf.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:29:33.638847Z","iopub.execute_input":"2022-08-06T15:29:33.640456Z","iopub.status.idle":"2022-08-06T15:29:53.942890Z","shell.execute_reply.started":"2022-08-06T15:29:33.640393Z","shell.execute_reply":"2022-08-06T15:29:53.941124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.Series(final_pred,name=\"Label\")","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:34:25.769038Z","iopub.execute_input":"2022-08-06T15:34:25.769612Z","iopub.status.idle":"2022-08-06T15:34:25.776372Z","shell.execute_reply.started":"2022-08-06T15:34:25.769575Z","shell.execute_reply":"2022-08-06T15:34:25.775223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results], axis=1)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:34:36.656905Z","iopub.execute_input":"2022-08-06T15:34:36.658359Z","iopub.status.idle":"2022-08-06T15:34:36.707343Z","shell.execute_reply.started":"2022-08-06T15:34:36.658277Z","shell.execute_reply":"2022-08-06T15:34:36.706319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}