{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport warnings\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nfrom scipy.stats import norm\nfrom sklearn.decomposition import PCA\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data \n","metadata":{}},{"cell_type":"code","source":"Data = pd.read_csv('../input/data-science-london-scikit-learn/train.csv',header=None)\nLabel = pd.read_csv('../input/data-science-london-scikit-learn/trainLabels.csv',header=None,names=['label'])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Describtion & Information About Data","metadata":{}},{"cell_type":"code","source":"Data.info()\nD=Data.copy(deep=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Correlation Between Label Column And Train Data ","metadata":{}},{"cell_type":"code","source":"x=Data.corrwith(Label['label'])\nx = np.asarray(x).reshape(40,1)\n\nfig, ax = plt.subplots(figsize=(10, 15))\nsns.heatmap(x, annot = True,cmap= 'Blues')#, fmt='.6g'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Replace Columns That Has Bad Correlation With Label as they Made Miss Leading by a Two New columns has Correlation one Positive And one Negative ","metadata":{}},{"cell_type":"code","source":"EC = Data.columns\nDrop_col=[]\nfor i in EC:\n    if(((Data[i].corr(Label['label']) < 0.01) and (Data[i].corr(Label['label'])> 0)) or \n    ((Data[i].corr(Label['label']) < 0) and (Data[i].corr(Label['label'])> -0.09))):\n        Drop_col.append(i)\n        Data.drop(i, axis=1, inplace=True)\n        \nEC=Data.columns\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using PCA To Make 2 New Columns By Combination Of Bad Correlation Columns ","metadata":{}},{"cell_type":"code","source":"x=D.iloc[:,Drop_col]\n\nmodel = PCA(n_components= 2)\nmodel.fit(x)\n\ndata = model.transform(x)\ndata = pd.DataFrame(data)\nData = pd.concat([Data, data], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x=Data.corrwith(Label['label'])\nx = np.asarray(x).reshape(26,1)\n\nfig, ax = plt.subplots(figsize=(10, 15))\nsns.heatmap(x, annot = True,cmap= 'Blues')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# First Graph Represent That The Maximun number of Label is 1\n### Second Graph represent that and specific The number of Unique Values in this Column is 2( 0 , 1 )","metadata":{}},{"cell_type":"code","source":"sns.distplot(Label['label'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(Label['label'], plot=plt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization\n### Get Columns Normal Graph To Make Sure That Data Has Normal Distribution With Density","metadata":{}},{"cell_type":"code","source":"horizontal_concat = pd.concat([Data, Label['label']], axis=1)\nhorizontal_concat=pd.DataFrame(horizontal_concat).astype('object')\nsns.pairplot(horizontal_concat, corner=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in Data.columns:\n    sns.distplot(Data[i], fit=norm);\n    fig = plt.figure()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Standard Scaler for Data","metadata":{}},{"cell_type":"code","source":"sc_X = StandardScaler()\nX_train = sc_X.fit_transform(Data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Out=Label['label']\nOut=np.array(Out)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Doing All Previous Data Preprocessing On Test Data","metadata":{}},{"cell_type":"code","source":"Test_Data = pd.read_csv('../input/data-science-london-scikit-learn/test.csv',header=None)\nTest_Data.info()\nD=Test_Data.copy(deep=True)\nTest_Data = Test_Data.loc[:,EC]\nx=D.iloc[:,Drop_col]\nmodel = PCA(n_components= 2)\nmodel.fit(x)\ndata = model.transform(x)\ndata = pd.DataFrame(data)\nTest_Data = pd.concat([Test_Data, data], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Test_Data.info()\nX_Test = sc_X.fit_transform(Test_Data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine Learning Modeling\n","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.linear_model import Perceptron\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import confusion_matrix\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params=[{'solver':('lbfgs','newton-cg','saga'),'penalty':('l2','l1','elasticnet')},\n        {'kernel':('linear', 'rbf'), 'C':[1, 10]},\n        {'penalty':('l2','l1','elasticnet'),'multi_class':('ovr', 'crammer_singer')},\n        {'max_depth':[3,5,10,15],'n_estimators':[50,100,150,200]},\n        {'algorithm':('auto', 'ball_tree', 'kd_tree', 'brute')},\n        {'alpha': [0.1,0.0001,1],'penalty':('l2','l1','elasticnet')},\n        {'max_depth':[3,5,10,15]}]\n\n\nLR = LogisticRegression(n_jobs=-1)\nSV = SVC()\nLS = LinearSVC(random_state=False)\nRR = RandomForestClassifier(random_state=False)\nKC = KNeighborsClassifier(n_neighbors = 5,weights= 'uniform')\nSC = SGDClassifier(shuffle=True,n_jobs=-1,random_state=False)\nDC = DecisionTreeClassifier(random_state=False)\n\nmodels=[LR,SV,LS,RR,KC,SC,DC]\npred=[]\n\nfor model,param in zip(models,params):\n    print('Model is ',model)\n    for i in range (3,11):\n        GridSearchModel = GridSearchCV(model,param, cv = i,return_train_score=True, n_jobs=-1)\n        GridSearchModel.fit(X_train,Out)\n        VD_predict=GridSearchModel.predict(X_Test)       \n        VDedict=GridSearchModel.predict(X_train)\n        cm = confusion_matrix(Out, VDedict)\n        pred.append((GridSearchModel.best_score_,cm,[VD_predict]))\n        sorted(GridSearchModel.cv_results_.keys())\n        print('CV = ',i)\n        print('Best Score is :', GridSearchModel.best_score_)\n        print('Best Parameters are :', GridSearchModel.best_params_)\n        print('confusion matrix = ')\n        print(cm)\n        print('-------------------------------')\n    print('\\t\\t*****************************************************************')\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The Best Score is 90 %","metadata":{}},{"cell_type":"code","source":"pred.sort(key=lambda y: y[0])\nsns.heatmap(pred[-1][1], center = True)\nplt.show()\nprediction=pred[-1][-1]\nprediction=np.array(prediction)\nprediction=prediction.reshape(9000,1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n","metadata":{"execution":{"iopub.status.busy":"2022-07-05T03:26:52.161961Z","iopub.execute_input":"2022-07-05T03:26:52.162408Z","iopub.status.idle":"2022-07-05T03:26:53.039813Z","shell.execute_reply.started":"2022-07-05T03:26:52.16237Z","shell.execute_reply":"2022-07-05T03:26:53.038718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(prediction)\nsubmission.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.columns = ['Solution']\nsubmission['Id'] = np.arange(1,submission.shape[0]+1)\nsubmission = submission[['Id', 'Solution']]\nsubmission.to_csv('submission_scikit_learn.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-05T03:26:55.719469Z","iopub.execute_input":"2022-07-05T03:26:55.719844Z","iopub.status.idle":"2022-07-05T03:26:55.738036Z","shell.execute_reply.started":"2022-07-05T03:26:55.719815Z","shell.execute_reply":"2022-07-05T03:26:55.737182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-05T03:26:58.993517Z","iopub.execute_input":"2022-07-05T03:26:58.993898Z","iopub.status.idle":"2022-07-05T03:26:59.002831Z","shell.execute_reply.started":"2022-07-05T03:26:58.993866Z","shell.execute_reply":"2022-07-05T03:26:59.001565Z"},"trusted":true},"execution_count":null,"outputs":[]}]}