{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport time\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/train.csv',skiprows=range(1,149903891),nrows=3500000)","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"a67c8071-7ab9-4546-a86f-2705fa6f7b3b","_uuid":"bc459195952280e530ee0c58d98e52ff5f828686","collapsed":true,"trusted":true},"cell_type":"code","source":"test=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e5f88bb6-4c2f-45b3-af2c-4238d6f1fc92","_uuid":"c15d15c8054ba7d6d8ba5246ed5b0b0c4059f344","trusted":true},"cell_type":"code","source":"#展示数据\ntrain.columns=['ip','app','device','os','channel','click_time','attributed_timed','is_attributed']\nprint(train.head())\nprint('*'*10)\n#print(test.head())","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"f52db577-e250-46b6-96c7-f18ea92742bb","_uuid":"34181008609e81e6052975b032978d1f579e78b2","trusted":true},"cell_type":"code","source":"#数据的统计信息\nprint(train['is_attributed'].value_counts())\nprint(train[train['is_attributed']==1]['is_attributed'].sum()/len(train))\n'''\n0    1992862\n1       7138\nName: is_attributed, dtype: int64\n0.003569\n\n调参时减小训练数据量\n'''","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"4f644b16-766d-42cf-91f3-02a2adcd012a","_uuid":"82b9ff173d11d65fecd342c3766ff4bbde7b4504","collapsed":true,"trusted":true},"cell_type":"code","source":"y=train['is_attributed']\n#'click_time','is_attributed','attributed_timed'\ntrain.drop(['click_time','is_attributed','attributed_timed'],axis=1,inplace=True)#inplace=True代表更改原内存的值\n#'click_id','click_time'\ntest.drop(['click_id','click_time'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"51c5dc4f-e1c8-488f-9bff-2b1df45a18d5","_uuid":"7fee71b9a074f3f4fddfcc8413bf14c220863208","collapsed":true,"trusted":true},"cell_type":"code","source":"def print_score(m,dt,y):\n    print('Accuracy:[Train,Val]')\n    res=[m.score(dt,y)]#验证得分\n    if hasattr(m,'obb_score_'):res.append(m.obb_score_)#袋外验证得分\n    print(res)\n    \n    print('Train Confusion Matrix')\n    df_train_proba=m.predict_proba(dt)#预测概率，原始的值\n    df_train_pre_indices=np.argmax(df_train_proba,axis=1)#找到每一行概率最大值的索引\n    print(df_train_pre_indices)\n    classes_train=np.unique(y)#类别个数\n    preds_train=classes_train[df_train_pre_indices]\n    print('*'*10)\n    print(preds_train)\n    skplt.plot_confusion_matrix(y,preds_train)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"73cfb326-4a2f-44b8-8cac-75f6385b832e","_uuid":"524f6c789bfd4f6d0351bb4533f697e3c47281a0","collapsed":true,"trusted":true},"cell_type":"code","source":"#提交数据集的模板\ntest_submission = pd.read_csv(\"../input/sample_submission.csv\")\ntest_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9f888731-9ff2-4dc1-968e-daa3f495ddcc","_uuid":"99b02b784cf7bbd38de8501a5e7c487941e2c177","collapsed":true,"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn import cross_validation,metrics\nfrom sklearn.grid_search import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e97b6bac-b128-4808-a0e2-a3be4e3bcf0e","_uuid":"941305e5da3ff6a853045b3d37762dd1fb5935fa","collapsed":true,"trusted":true},"cell_type":"code","source":"gbm4= GradientBoostingClassifier(learning_rate=0.1,n_estimators=100,max_depth=5, min_samples_leaf =90,\n                                 min_samples_split =2,max_features=2, subsample=0.75, random_state=10)  \n%time gbm4.fit(train,y)  \n#y_pred= gbm4.predict(train)  \ntest_submission['is_attributed']= gbm4.predict_proba(test)[:,1]  \n#print(\"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)  )\n#print(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)   )\n'''\nAccuracy : 0.9973\nAUC Score (Train): 0.957531\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"13d915d9-28c6-490b-9d90-d7405adc2569","_uuid":"1082c8fca85caffd803140cde7756f156c5ab93c","collapsed":true,"trusted":true},"cell_type":"code","source":"#保存结果\ntest_submission.to_csv('rf4_result.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"aa4f94be-f605-4f9f-b9a9-745c84b1f6aa","_uuid":"c4d8faa3d58733d295268fab56becf51cfdf47bf","collapsed":true,"trusted":true},"cell_type":"code","source":"'''gbm0= GradientBoostingClassifier(random_state=10)  \n%time gbm0.fit(train,y)  \n#y_pred= gbm0.predict(X)  \n#y_predprob= gbm0.predict_proba(X)[:,1]  #预测概率\n\nGradientBoostingClassifier(criterion='friedman_mse', init=None,\n              learning_rate=0.1, loss='deviance', max_depth=3,\n              max_features=None, max_leaf_nodes=None,\n              min_impurity_decrease=0.0, min_impurity_split=None,\n              min_samples_leaf=1, min_samples_split=2,\n              min_weight_fraction_leaf=0.0, n_estimators=100,\n              presort='auto', random_state=10, subsample=1.0, verbose=0,\n              warm_start=False)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d77f0fd5-d94f-445b-a024-08a37334a015","_uuid":"e280b0b0e0286b13c6e543a80904ed2f255d7e7d","collapsed":true,"trusted":true},"cell_type":"code","source":"'''y_pred= gbm1.predict(train)  \ny_predprob= gbm1.predict_proba(train)[:,1]  \nprint(\"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)  )\nprint(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)  )\n\nAccuracy : 0.9975\nAUC Score (Train): 0.934203\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b4edfba8-716d-466f-8694-73796bcee45c","_uuid":"0239ad6f45c7cebcb81d67299a8f5a2dfdb35d5f","collapsed":true,"trusted":true},"cell_type":"code","source":"'''#调参来提高模型泛化能力\nparam_test1= {'n_estimators':[_ for _ in range(80,120,10)]}  \ngsearch1= GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,min_samples_split=300,min_samples_leaf=20,max_depth=8,\n                     max_features='sqrt',subsample=0.8,random_state=10), param_grid= param_test1, scoring='roc_auc',iid=False,cv=5)  \ngsearch1.fit(train,y)  \ngsearch1.grid_scores_,gsearch1.best_params_, gsearch1.best_score_  \n\n([mean: 0.93674, std: 0.01494, params: {'n_estimators': 80},\n  mean: 0.93646, std: 0.01580, params: {'n_estimators': 90},\n  mean: 0.93795, std: 0.01442, params: {'n_estimators': 100},\n  mean: 0.93787, std: 0.01436, params: {'n_estimators': 110}],\n {'n_estimators': 100},\n 0.9379545746007757)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"143a5dba-a58f-4ac0-a384-49f31665f1bb","_uuid":"5bfca136a29597cd5a5e17f6a2286a89b0d932b4","collapsed":true,"trusted":true},"cell_type":"code","source":"'''#对max_depth进行调参\nparam_test2= {'max_depth':[_ for _ in range(3,14,2)], 'min_samples_split':[_ for _ in range(2,8,2)]}  \ngsearch2= GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,n_estimators=100, min_samples_leaf=20, max_features='sqrt', subsample=0.8,random_state=10),  \nparam_grid= param_test2,  scoring='roc_auc',  iid=False,  cv=5)  \ngsearch2.fit(train,y)  \ngsearch2.grid_scores_,gsearch2.best_params_, gsearch2.best_score_  \n\n([mean: 0.91991, std: 0.01615, params: {'max_depth': 3, 'min_samples_split': 2},\n  mean: 0.91991, std: 0.01615, params: {'max_depth': 3, 'min_samples_split': 4},\n  mean: 0.91991, std: 0.01615, params: {'max_depth': 3, 'min_samples_split': 6},\n  mean: 0.93837, std: 0.01390, params: {'max_depth': 5, 'min_samples_split': 2},\n  mean: 0.93837, std: 0.01390, params: {'max_depth': 5, 'min_samples_split': 4},\n  mean: 0.93837, std: 0.01390, params: {'max_depth': 5, 'min_samples_split': 6},\n  mean: 0.89703, std: 0.03756, params: {'max_depth': 7, 'min_samples_split': 2},\n  mean: 0.89703, std: 0.03756, params: {'max_depth': 7, 'min_samples_split': 4},\n  mean: 0.89703, std: 0.03756, params: {'max_depth': 7, 'min_samples_split': 6},\n  mean: 0.87293, std: 0.02422, params: {'max_depth': 9, 'min_samples_split': 2},\n  mean: 0.87293, std: 0.02422, params: {'max_depth': 9, 'min_samples_split': 4},\n  mean: 0.87293, std: 0.02422, params: {'max_depth': 9, 'min_samples_split': 6},\n  mean: 0.84115, std: 0.05386, params: {'max_depth': 11, 'min_samples_split': 2},\n  mean: 0.84115, std: 0.05386, params: {'max_depth': 11, 'min_samples_split': 4},\n  mean: 0.84115, std: 0.05386, params: {'max_depth': 11, 'min_samples_split': 6},\n  mean: 0.88446, std: 0.04320, params: {'max_depth': 13, 'min_samples_split': 2},\n  mean: 0.88446, std: 0.04320, params: {'max_depth': 13, 'min_samples_split': 4},\n  mean: 0.88446, std: 0.04320, params: {'max_depth': 13, 'min_samples_split': 6}],\n {'max_depth': 5, 'min_samples_split': 2},\n 0.9383689834014476)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b0ee502c-8d41-45b1-b26e-996814540237","_uuid":"38ec097e8e0d11cd45f4c6a4720daa2401fd1d41","collapsed":true,"trusted":true},"cell_type":"code","source":"'''#min_samples_leaf一起调参。  \nparam_test3= {'min_samples_split':[_ for _ in range(2,8,2)],'min_samples_leaf':[_ for _ in range(60,101,10)]}  \ngsearch3= GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,n_estimators=100,max_depth=5,max_features='sqrt',subsample=0.8,random_state=10),  \nparam_grid= param_test3,  scoring='roc_auc',  iid=False,  cv=5,n_jobs=-1)  \ngsearch3.fit(train,y)  \ngsearch3.grid_scores_,gsearch3.best_params_, gsearch3.best_score_ \n\n([mean: 0.93795, std: 0.00871, params: {'min_samples_leaf': 60, 'min_samples_split': 2},\n  mean: 0.93795, std: 0.00871, params: {'min_samples_leaf': 60, 'min_samples_split': 4},\n  mean: 0.93795, std: 0.00871, params: {'min_samples_leaf': 60, 'min_samples_split': 6},\n  mean: 0.93666, std: 0.01605, params: {'min_samples_leaf': 70, 'min_samples_split': 2},\n  mean: 0.93666, std: 0.01605, params: {'min_samples_leaf': 70, 'min_samples_split': 4},\n  mean: 0.93666, std: 0.01605, params: {'min_samples_leaf': 70, 'min_samples_split': 6},\n  mean: 0.93552, std: 0.01466, params: {'min_samples_leaf': 80, 'min_samples_split': 2},\n  mean: 0.93552, std: 0.01466, params: {'min_samples_leaf': 80, 'min_samples_split': 4},\n  mean: 0.93552, std: 0.01466, params: {'min_samples_leaf': 80, 'min_samples_split': 6},\n  mean: 0.93907, std: 0.01263, params: {'min_samples_leaf': 90, 'min_samples_split': 2},\n  mean: 0.93907, std: 0.01263, params: {'min_samples_leaf': 90, 'min_samples_split': 4},\n  mean: 0.93907, std: 0.01263, params: {'min_samples_leaf': 90, 'min_samples_split': 6},\n  mean: 0.93874, std: 0.01164, params: {'min_samples_leaf': 100, 'min_samples_split': 2},\n  mean: 0.93874, std: 0.01164, params: {'min_samples_leaf': 100, 'min_samples_split': 4},\n  mean: 0.93874, std: 0.01164, params: {'min_samples_leaf': 100, 'min_samples_split': 6}],\n {'min_samples_leaf': 90, 'min_samples_split': 2},\n 0.9390701404941414)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"55f7e1ea-1982-48c1-9119-54e91b0f3402","_uuid":"c6a5fc58d1e377b9f36085f6d1a37a898c2ea9ec","collapsed":true,"trusted":true},"cell_type":"code","source":"'''#用调优参数估计预测\ngbm1= GradientBoostingClassifier(learning_rate=0.1, n_estimators=100,max_depth=5,min_samples_leaf =90, min_samples_split =2, \n                                 max_features='sqrt',subsample=0.8, random_state=10)  \ngbm1.fit(train,y)  \ny_pred= gbm1.predict(train)  \ny_predprob= gbm1.predict_proba(train)[:,1]  \nprint(\"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)  )\nprint(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)   )\n\nAccuracy : 0.9972\nAUC Score (Train): 0.952916\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"94e7132d-e48a-4ee2-8225-730e79a4c81b","_uuid":"3c2492fe0d8a375a4844e1adedc83f1d7511658a","collapsed":true,"trusted":true},"cell_type":"code","source":"#train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6b6dba32-26cb-43c3-a55f-9ac22e579ef0","_uuid":"2e06d7601ac1859b7c6426a7650fd6b99ec9a65d","collapsed":true,"trusted":true},"cell_type":"code","source":"'''param_test4= {'max_features':[_ for _ in range(1,6,1)]}  \ngsearch4= GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,n_estimators=100,max_depth=5, min_samples_leaf =90, min_samples_split =2,subsample=0.8, random_state=10),  \nparam_grid= param_test4,  scoring='roc_auc',  iid=False,  cv=5,n_jobs=-1)  \ngsearch4.fit(train,y)  \ngsearch4.grid_scores_,gsearch4.best_params_, gsearch4.best_score_   \n\n([mean: 0.93640, std: 0.01683, params: {'max_features': 1},\n  mean: 0.93907, std: 0.01263, params: {'max_features': 2},\n  mean: 0.93895, std: 0.01413, params: {'max_features': 3},\n  mean: 0.88630, std: 0.06103, params: {'max_features': 4},\n  mean: 0.91486, std: 0.02613, params: {'max_features': 5}],\n {'max_features': 2},\n 0.9390701404941414)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"04bfb4ee-2657-4a22-893e-3f73704075c7","_uuid":"3ee545352b7f59ea783039f5734ee6082ae947d9","collapsed":true,"trusted":true},"cell_type":"code","source":"'''param_test5= {'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}  \ngsearch5= GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,n_estimators=100,max_depth=5, min_samples_leaf =90, \n min_samples_split =2,max_features=2, random_state=10),  param_grid= param_test5,  scoring='roc_auc',  iid=False,   cv=5,n_jobs=-1)  \ngsearch5.fit(train,y)  \ngsearch5.grid_scores_,gsearch5.best_params_, gsearch5.best_score_   \n\n([mean: 0.93699, std: 0.00974, params: {'subsample': 0.6},\n  mean: 0.93832, std: 0.01243, params: {'subsample': 0.7},\n  mean: 0.93926, std: 0.01257, params: {'subsample': 0.75},\n  mean: 0.93907, std: 0.01263, params: {'subsample': 0.8},\n  mean: 0.93739, std: 0.01086, params: {'subsample': 0.85},\n  mean: 0.92796, std: 0.01406, params: {'subsample': 0.9}],\n {'subsample': 0.75},\n 0.9392626480478425)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b36e1d67-c9e0-4a1c-a075-a937a6a7665c","_uuid":"767cb2267e9d0d99ff89c493955d3311349a5543","collapsed":true,"trusted":true},"cell_type":"code","source":"'''#这时我们可以减半步长，最大迭代次数加倍来增加我们模型的泛化能力。再次拟合我们的模型：\ngbm2= GradientBoostingClassifier(learning_rate=0.05, n_estimators=200,max_depth=5,min_samples_leaf =90, min_samples_split =2,\n                                 max_features=2, subsample=0.75,random_state=10)  \ngbm2.fit(train,y)  \ny_pred= gbm2.predict(train)  \ny_predprob= gbm2.predict_proba(train)[:,1]  \nprint(\"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)  )\nprint(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)   )\n\nAccuracy : 0.9972\nAUC Score (Train): 0.953702\n'''\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0a9a5b8e-98bf-42b6-a7ab-3e2e7c5e2788","_uuid":"c3ff8f9ee9aa7dfb884607b2a848ada8412bfb7b","collapsed":true,"trusted":true},"cell_type":"code","source":"'''gbm3= GradientBoostingClassifier(learning_rate=0.01, n_estimators=1000,max_depth=5,min_samples_leaf =90,  \n               min_samples_split =2,max_features=2, subsample=0.75, random_state=10)  \ngbm3.fit(train,y)  \ny_pred= gbm3.predict(train)  \ny_predprob= gbm3.predict_proba(train)[:,1]  \nprint(\"Accuracy : %.4g\" % metrics.accuracy_score(y.values, y_pred)  )\nprint(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, y_predprob)   )\n\nAccuracy : 0.9973\nAUC Score (Train): 0.956455\n'''","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}