{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.stats import norm\nfrom sklearn.decomposition import PCA\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T06:08:45.199669Z","iopub.execute_input":"2022-07-06T06:08:45.200235Z","iopub.status.idle":"2022-07-06T06:08:45.207898Z","shell.execute_reply.started":"2022-07-06T06:08:45.200202Z","shell.execute_reply":"2022-07-06T06:08:45.206987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data \n","metadata":{}},{"cell_type":"code","source":"Data = pd.read_csv('/kaggle/input/santander-customer-transaction-prediction/train.csv')\nData.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:45.214009Z","iopub.execute_input":"2022-07-06T06:08:45.215028Z","iopub.status.idle":"2022-07-06T06:08:51.729545Z","shell.execute_reply.started":"2022-07-06T06:08:45.214990Z","shell.execute_reply":"2022-07-06T06:08:51.728492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Describtion & Information About Data","metadata":{}},{"cell_type":"code","source":"Data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:51.731599Z","iopub.execute_input":"2022-07-06T06:08:51.732309Z","iopub.status.idle":"2022-07-06T06:08:51.751962Z","shell.execute_reply.started":"2022-07-06T06:08:51.732269Z","shell.execute_reply":"2022-07-06T06:08:51.750867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:51.753436Z","iopub.execute_input":"2022-07-06T06:08:51.754321Z","iopub.status.idle":"2022-07-06T06:08:53.665279Z","shell.execute_reply.started":"2022-07-06T06:08:51.754282Z","shell.execute_reply":"2022-07-06T06:08:53.664220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data=Data.drop('ID_code',axis='columns')\nD=Data.copy(deep=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:53.671316Z","iopub.execute_input":"2022-07-06T06:08:53.672732Z","iopub.status.idle":"2022-07-06T06:08:53.925749Z","shell.execute_reply.started":"2022-07-06T06:08:53.672689Z","shell.execute_reply":"2022-07-06T06:08:53.924639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Correlation Between Target Column And Train Data \n\n### Replace Columns That Has Bad (Small) Correlation With Target as they Made Miss Leading by a Two New columns has Correlation one Positive And one Negative ","metadata":{}},{"cell_type":"code","source":"EC = Data.columns\nDrop_col=[]\nfor i in EC:\n    #print(i)\n    if(((Data[i].corr(Data['target']) < 0.01) and (Data[i].corr(Data['target'])> 0)) or \n    ((Data[i].corr(Data['target']) < 0) and (Data[i].corr(Data['target'])> -0.01))):\n        Drop_col.append((i,Data[i].corr(Data['target']) ))\n        Data.drop(i, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:53.930259Z","iopub.execute_input":"2022-07-06T06:08:53.930807Z","iopub.status.idle":"2022-07-06T06:08:58.744030Z","shell.execute_reply.started":"2022-07-06T06:08:53.930768Z","shell.execute_reply":"2022-07-06T06:08:58.742922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using PCA To Make 3 New Columns By Combination Of Bad Correlation Columns ","metadata":{}},{"cell_type":"code","source":"EC=Data.columns\nDrop_col\nColumn_Name=[x[0] for x in Drop_col]\nx=D.loc[:,Column_Name]\n\nmodel = PCA(n_components= 3, svd_solver='full')#it can be full,arpack,randomized\nmodel.fit(x)\n\ndata = model.transform(x)\ndata = pd.DataFrame(data)\n\nData = pd.concat([Data, data], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:58.745628Z","iopub.execute_input":"2022-07-06T06:08:58.746178Z","iopub.status.idle":"2022-07-06T06:08:59.550873Z","shell.execute_reply.started":"2022-07-06T06:08:58.746138Z","shell.execute_reply":"2022-07-06T06:08:59.549916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:59.552287Z","iopub.execute_input":"2022-07-06T06:08:59.552662Z","iopub.status.idle":"2022-07-06T06:08:59.573339Z","shell.execute_reply.started":"2022-07-06T06:08:59.552627Z","shell.execute_reply":"2022-07-06T06:08:59.572223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization\n\n### The Graph Show that Maximum Category oF Target Is 0 ","metadata":{}},{"cell_type":"code","source":"sns.countplot(Data['target'])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:59.575207Z","iopub.execute_input":"2022-07-06T06:08:59.575642Z","iopub.status.idle":"2022-07-06T06:08:59.732974Z","shell.execute_reply.started":"2022-07-06T06:08:59.575596Z","shell.execute_reply":"2022-07-06T06:08:59.731924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get Columns Normal Graph To Make Sure That Data Has Normal Distribution With Density","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(13,13,figsize=(22,33))\nz=1\nfor i in Data.columns:\n    if i == 'target':\n        continue\n        \n    plt.subplot(14,13,z)\n    sns.distplot(Data[i], fit=norm);\n    z=z+1","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:08:59.734689Z","iopub.execute_input":"2022-07-06T06:08:59.735376Z","iopub.status.idle":"2022-07-06T06:11:46.051557Z","shell.execute_reply.started":"2022-07-06T06:08:59.735339Z","shell.execute_reply":"2022-07-06T06:11:46.050704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spliting And Standard Scaler for Data  ","metadata":{}},{"cell_type":"code","source":"\nX=Data.iloc[:,1:]\ny=Data.iloc[:,0]\nsc_X = StandardScaler()\nX_train = sc_X.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:11:46.057362Z","iopub.execute_input":"2022-07-06T06:11:46.057920Z","iopub.status.idle":"2022-07-06T06:11:46.731164Z","shell.execute_reply.started":"2022-07-06T06:11:46.057883Z","shell.execute_reply":"2022-07-06T06:11:46.730184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Doing All Previous Data Preprocessing On Test Data","metadata":{}},{"cell_type":"code","source":"Test_Data = pd.read_csv('/kaggle/input/santander-customer-transaction-prediction/test.csv')\nTest_Data=Test_Data.drop('ID_code',axis='columns')\nEC=EC[1:]\nTest_Data.info()\nD=Test_Data.copy(deep=True)\nTest_Data = Test_Data.loc[:,EC]\nTest_Data.info()\n\nx=D.loc[:,Column_Name]\nmodel = PCA(n_components= 3)\nmodel.fit(x)\ndata = model.transform(x)\ndata = pd.DataFrame(data)\nTest_Data = pd.concat([Test_Data, data], axis=1)\n\nX_test = sc_X.fit_transform(Test_Data)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:11:46.732665Z","iopub.execute_input":"2022-07-06T06:11:46.733130Z","iopub.status.idle":"2022-07-06T06:11:55.884290Z","shell.execute_reply.started":"2022-07-06T06:11:46.733083Z","shell.execute_reply":"2022-07-06T06:11:55.883347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine Learning Models\n","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import  LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import confusion_matrix\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:11:55.885641Z","iopub.execute_input":"2022-07-06T06:11:55.886612Z","iopub.status.idle":"2022-07-06T06:11:55.894145Z","shell.execute_reply.started":"2022-07-06T06:11:55.886559Z","shell.execute_reply":"2022-07-06T06:11:55.893086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nY_train=y\nlogreg = LogisticRegression(random_state=False, solver= 'newton-cg')\nlogreg.fit(X_train, Y_train)\nY_pred1 = logreg.predict(X_test)\nacc_log = round(logreg.score(X_train, Y_train) * 100, 2)\nacc_log\nprint('Best Score is :',acc_log)\nVDedict=logreg.predict(X_train)\ncm = confusion_matrix(y, VDedict)\nprint('confusion matrix = \\n',cm)\nsns.heatmap(cm, center = True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:11:55.895372Z","iopub.execute_input":"2022-07-06T06:11:55.896376Z","iopub.status.idle":"2022-07-06T06:12:01.518630Z","shell.execute_reply.started":"2022-07-06T06:11:55.896338Z","shell.execute_reply":"2022-07-06T06:12:01.517472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_forest = RandomForestClassifier(n_estimators=150,random_state=False,max_depth=15)\nrandom_forest.fit(X_train, Y_train)\nY_pred2 = random_forest.predict(X_test)\nrandom_forest.score(X_train, Y_train)\nacc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)\nacc_random_forest\nprint('Best Score is :',acc_random_forest)\nVDedict=random_forest.predict(X_train)\ncm = confusion_matrix(y, VDedict)\nprint('confusion matrix = \\n',cm)\nsns.heatmap(cm, center = True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:12:01.523403Z","iopub.execute_input":"2022-07-06T06:12:01.526446Z","iopub.status.idle":"2022-07-06T06:19:53.761374Z","shell.execute_reply.started":"2022-07-06T06:12:01.526410Z","shell.execute_reply":"2022-07-06T06:19:53.760315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ndecision_tree = DecisionTreeClassifier(random_state=False)\ndecision_tree.fit(X_train, Y_train)\nY_pred3 = decision_tree.predict(X_test)\nacc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)\nacc_decision_tree\nprint('Best Score is :',acc_decision_tree)\nVDedict=decision_tree.predict(X_train)\ncm = confusion_matrix(y, VDedict)\nprint('confusion matrix = \\n',cm)\nsns.heatmap(cm, center = True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:19:53.766899Z","iopub.execute_input":"2022-07-06T06:19:53.769442Z","iopub.status.idle":"2022-07-06T06:23:29.293688Z","shell.execute_reply.started":"2022-07-06T06:19:53.769398Z","shell.execute_reply":"2022-07-06T06:23:29.292696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"linear_svc = LinearSVC(random_state=False)\nlinear_svc.fit(X_train, Y_train)\nY_pred5 = linear_svc.predict(X_test)\nacc_linear_svc = round(linear_svc.score(X_train, Y_train) * 100, 2)\nacc_linear_svc\nprint('Best Score is :',acc_linear_svc)\nVDedict=linear_svc.predict(X_train)\ncm = confusion_matrix(y, VDedict)\nprint('confusion matrix = \\n',cm)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:23:29.295234Z","iopub.execute_input":"2022-07-06T06:23:29.295622Z","iopub.status.idle":"2022-07-06T06:26:30.653303Z","shell.execute_reply.started":"2022-07-06T06:23:29.295586Z","shell.execute_reply":"2022-07-06T06:26:30.652318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgd = SGDClassifier()\nsgd.fit(X_train, Y_train)\nY_pred6 = sgd.predict(X_test)\nacc_sgd = round(sgd.score(X_train, Y_train) * 100, 2)\nacc_sgd\nprint('Best Score is :',acc_sgd)\nVDedict=sgd.predict(X_train)\ncm = confusion_matrix(y, VDedict)\nprint('confusion matrix = \\n',cm)\nsns.heatmap(cm, center = True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:26:30.654956Z","iopub.execute_input":"2022-07-06T06:26:30.655341Z","iopub.status.idle":"2022-07-06T06:26:34.645611Z","shell.execute_reply.started":"2022-07-06T06:26:30.655306Z","shell.execute_reply":"2022-07-06T06:26:34.644781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***You Can Use GridSearchCV IF You Want And The Code below Run Without Any Problem Just Remove Comment From Code 😊***","metadata":{}},{"cell_type":"code","source":"'''\nparams=[{'solver':('lbfgs','newton-cg','saga'),'penalty':('l2','l1','elasticnet')},\n        {'penalty':('l2','l1')},\n        {'max_depth':[3,5,10,15]},\n        {'algorithm':('auto', 'ball_tree', 'kd_tree', 'brute')},\n        {'alpha': [0.1,0.0001,1],'penalty':('l2','l1','elasticnet')},\n        {'max_depth':[3,5,10,15]}]\n\n\nLR = LogisticRegression(n_jobs=-1)\nLS = LinearSVC(random_state=False)\nRFR = RandomForestClassifier(n_estimators=100,random_state=False)\nKC = KNeighborsClassifier(n_neighbors = 5,weights= 'uniform')\nSC = SGDClassifier(shuffle=True,n_jobs=-1,random_state=False)\nDTC = DecisionTreeClassifier(random_state=False)\n\nmodels=[LR,LS,RFR,KC,SC,DTC]\n\n\nfor model,param in zip(models,params):\n    print('Model is ',model)\n    for i in range (3,11):\n        GridSearchModel = GridSearchCV(model,param, cv = i,return_train_score=True, n_jobs=-1)\n        GridSearchModel.fit(X, y)\n        VD_predict=GridSearchModel.predict(X_Test)\n        cm = confusion_matrix(VD, VD_predict)\n        sorted(GridSearchModel.cv_results_.keys())\n        print('CV = ',i)\n        print('Best Score is :', GridSearchModel.best_score_)\n        print('Best Parameters are :', GridSearchModel.best_params_)\n        print('confusion matrix = ')\n        print(cm)\n        print('-------------------------------')\n    print('\\t\\t*****************************************************************')\n\n'''","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:26:34.646902Z","iopub.execute_input":"2022-07-06T06:26:34.648037Z","iopub.status.idle":"2022-07-06T06:26:34.657690Z","shell.execute_reply.started":"2022-07-06T06:26:34.647999Z","shell.execute_reply":"2022-07-06T06:26:34.656782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/santander-customer-transaction-prediction/test.csv')\nsub_df = pd.DataFrame({\"ID_code\": df_test.ID_code.values})\nsub_df[\"target\"] = Y_pred3\nsub_df[:10]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:26:34.660769Z","iopub.execute_input":"2022-07-06T06:26:34.661042Z","iopub.status.idle":"2022-07-06T06:26:41.597512Z","shell.execute_reply.started":"2022-07-06T06:26:34.661011Z","shell.execute_reply":"2022-07-06T06:26:41.596498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T06:26:41.599084Z","iopub.execute_input":"2022-07-06T06:26:41.599543Z","iopub.status.idle":"2022-07-06T06:26:41.947671Z","shell.execute_reply.started":"2022-07-06T06:26:41.599506Z","shell.execute_reply":"2022-07-06T06:26:41.946617Z"},"trusted":true},"execution_count":null,"outputs":[]}]}