{"cells":[{"metadata":{},"cell_type":"markdown","source":"this notebook is edit of kernel \nhttps://www.kaggle.com/ks2019/siim-isic-notebook-0-9565-submission\nplease upvote him i only add hyperparametr tunning part ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install xgboost\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.datasets import load_iris\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train= pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest= pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nsub= pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['sex'] = train['sex'].fillna('na')\ntrain['age_approx'] = train['age_approx'].fillna(0)\ntrain['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].fillna('na')\n\ntest['sex'] = test['sex'].fillna('na')\ntest['age_approx'] = test['age_approx'].fillna(0)\ntest['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].fillna('na')\ntrain['sex'] = train['sex'].astype(\"category\").cat.codes +1\ntrain['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].astype(\"category\").cat.codes +1\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['sex'] = test['sex'].astype(\"category\").cat.codes +1\ntest['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].astype(\"category\").cat.codes +1\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = train[['sex', 'age_approx','anatom_site_general_challenge']]\ny_train = train['target']\n\n\nx_test = test[['sex', 'age_approx','anatom_site_general_challenge']]\n# y_train = test['target']\n\n\ntrain_DMatrix = xgb.DMatrix(x_train, label= y_train)\ntest_DMatrix = xgb.DMatrix(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = xgb.XGBClassifier(n_estimators=2600, \n                        max_depth=20, \n                        objective='multi:softprob',\n                        seed=0,  \n                        nthread=-1, \n                        learning_rate=0.15, \n                        num_class = 2, \n                        scale_pos_weight = (32542/584))\n\nclf.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.predict_proba(x_test)[:,1]\n# clf.predict(x_test)\nsub.target = clf.predict_proba(x_test)[:,1]\nsub_tabular = sub.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub_tabular.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Ensemble for 0.9565\n# sub_new = pd.read_csv('../input/siimisic/sub-new.csv')\n# sub_mean = pd.read_csv('/kaggle/input/siimisic/submission_mean.csv')\n# sub.target = sub_mean.target *0.2 + sub_new.target *0.6 + sub_tabular.target *0.2\n# sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_tabular","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ens_sub = pd.read_csv('../input/actual-output/submission.csv')\n#ens_sub.to_csv('ens_sub.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['target'] = (ens_sub.target - sub_tabular.target*0.2)/0.8\nsub.to_csv('efn_sub.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# xgboost cross validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''from scipy import stats\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import RandomizedSearchCV, KFold\nfrom sklearn.metrics import f1_score\n\nclf_xgb = XGBClassifier(objective = 'binary:logistic')\nparam_dist = {'n_estimators': stats.randint(150, 500),\n              'learning_rate': stats.uniform(0.01, 0.07),\n              'subsample': stats.uniform(0.3, 0.7),\n              'max_depth': [3, 4, 5, 6, 7, 8, 9],\n              'colsample_bytree': stats.uniform(0.5, 0.45),\n              'min_child_weight': [1, 2, 3]\n             }\nclf = RandomizedSearchCV(clf_xgb, param_distributions = param_dist, n_iter = 25, scoring = 'f1', error_score = 0, verbose = 3, n_jobs = -1)\n\nnumFolds = 5\nfolds = KFold(n_splits = numFolds, shuffle = True)\n\nestimators = []\nresults = np.zeros(len(x_train))\nscore = 0.0\nfor train_index, test_index in folds.split(x_train):\n    X_train, X_test = x_train[train_index,:], x_train.iloc[test_index,:]\n    y_train, y_test = y.iloc[train_index].values.ravel(), y.iloc[test_index].values.ravel()\n    clf.fit(X_train, y_train)\n\n    estimators.append(clf.best_estimator_)\n    results[test_index] = clf.predict(X_test)\n    score += f1_score(y_test, results[test_index])\nscore /= numFolds'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy import stats\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import RandomizedSearchCV, KFold\nfrom sklearn.metrics import f1_score\n\nclf_xgb = XGBClassifier(objective = 'multi:softprob')\nparam_dist = {'n_estimators': stats.randint(150, 500),\n              'learning_rate': stats.uniform(0.01, 0.07),\n              'subsample': stats.uniform(0.3, 0.7),\n               'num_class':  (2,3),\n              'max_depth': [3, 4, 5, 6, 7, 8, 9],\n              'colsample_bytree': stats.uniform(0.5, 0.45),\n              'min_child_weight': [1, 2, 3]\n             }\nclf = RandomizedSearchCV(clf_xgb, param_distributions = param_dist, n_iter = 25, scoring = 'f1', error_score = 0, verbose = 3, n_jobs = -1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.best_params_\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf1 = xgb.XGBClassifier(n_estimators=432, \n                        max_depth=7, \n                        objective='multi:softprob',\n                        seed=0,  \n                        nthread=-1, \n                        learning_rate=0.015573109334896062,\n                        num_class = 2, \n                        scale_pos_weight = (32542/584))\n\nclf1.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.predict_proba(x_test)[:,1]\n# clf.predict(x_test)\nsub.target = clf.predict_proba(x_test)[:,1]\nsub_tabular = sub.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_tabular.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''import numpy as np\nimport pandas as pd\nimport xgboost as xgb\n#from xgboost.sklearn import  XGBClassifier\nfrom sklearn.preprocessing import LabelEncoder\n#from sklearn import cross_validation, metrics\nfrom sklearn.model_selection import cross_validate\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV\n#from sklearn.grid_search import GridSearchCV\n\n\ndef modelFit(alg, X, y, useTrainCV=True, cvFolds=5, early_stopping_rounds=50):\n    if useTrainCV:\n        #xgbParams = alg.get_xgb_params()\n        xgTrain = xgb.DMatrix(X, label=y)\n        cvresult = xgb.cv(xgbParams,\n                      xgTrain,\n                      num_boost_round=alg.get_params()['n_estimators'],\n                      nfold=cvFolds,\n                      stratified=True,\n                      metrics={'mlogloss'},\n                      early_stopping_rounds=early_stopping_rounds,\n                      seed=0,\n                      callbacks=[xgb.callback.print_evaluation(show_stdv=False),                                                               xgb.callback.early_stop(3)])\n\n        print(cvresult)\n        alg.set_params(n_estimators=cvresult.shape[0])\n\n    # Fit the algorithm\n    alg.fit(X, y, eval_metric='mlogloss')\n\n    # Predict\n    dtrainPredictions = alg.predict(X)\n    dtrainPredProb = alg.predict_proba(X)\n\n    # Print model report:\n    print(\"\\nModel Report\")\n    print(\"Classification report: \\n\")\n    print(classification_report(y_val, y_val_pred))\n    print(\"Accuracy : %.4g\" % metrics.accuracy_score(y, dtrainPredictions))\n    print(\"Log Loss Score (Train): %f\" % metrics.log_loss(y, dtrainPredProb))\n    feat_imp = pd.Series(alg.booster().get_fscore()).sort_values(ascending=False)\n    feat_imp.plot(kind='bar', title='Feature Importances')\n    plt.ylabel('Feature Importance Score')\n\n\n# 1) Read training set\nprint('>> Read training set')\n#train = pd.read_csv(trainFile)\n\n# 2) Extract target attribute and convert to numeric\nprint('>> Preprocessing')\ny_train = y_train\n#train['OutcomeType'].values\nle_y = LabelEncoder()\ny_train = le_y.fit_transform(y_train)\n#train.drop('OutcomeType', axis=1, inplace=True)\n\n# 4) Extract features and target from training set\nX_train = x_train\n\n# 5) First classifier\nxgb = xgb(learning_rate =0.1,\n                    n_estimators=1000,\n                    max_depth=5,\n                    min_child_weight=1,\n                    gamma=0,\n                    subsample=0.8,\n                    colsample_bytree=0.8,\n                    scale_pos_weight=1,\n                    objective='multi:softprob',\n                    seed=27)\n\nmodelFit(xgb, X_train, y_train)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train1,  x_test1, y_train1, y_test1=train_test_split(x_train, y_train, test_size=0.3, random_state=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtrain = xgb.DMatrix(x_train1, label=y_train1)\ndtest = xgb.DMatrix(x_test1, label=y_test1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nimport numpy as np\n# \"Learn\" the mean from the training data\nmean_train = np.mean(y_train)\n# Get predictions on the test set\nbaseline_predictions = np.ones(y_test1.shape) * mean_train\n# Compute MAE\nmae_baseline = mean_absolute_error(y_test1, baseline_predictions)\nprint(\"Baseline MAE is {:.2f}\".format(mae_baseline))\n#Baseline MAE is 11.31","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\n    # Parameters that we are going to tune.\n    'max_depth':6,\n    'min_child_weight': 1,\n    'eta':.3,\n    'subsample': 1,\n    'colsample_bytree': 1,\n    # Other parameters\n    'objective':'reg:linear',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params['eval_metric'] = \"mae\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_boost_round = 999","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"model = xgb.train(\n    params,\n    dtrain,\n    num_boost_round=num_boost_round,\n    evals=[(dtest, \"Test\")],\n    early_stopping_rounds=10\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best MAE: {:.2f} with {} rounds\".format(\n                 model.best_score,\n                 model.best_iteration+1))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"cv_results = xgb.cv(\n    params,\n    dtrain,\n    num_boost_round=num_boost_round,\n    seed=42,\n    nfold=5,\n    metrics={'mae'},\n    early_stopping_rounds=10\n)\ncv_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv_results['test-mae-mean'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gridsearch_params = [\n    (max_depth, min_child_weight)\n    for max_depth in range(9,12)\n    for min_child_weight in range(5,8)\n]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Define initial best params and MAE\nmin_mae = float(\"Inf\")\nbest_params = None\nfor max_depth, min_child_weight in gridsearch_params:\n    print(\"CV with max_depth={}, min_child_weight={}\".format(\n                             max_depth,\n                             min_child_weight))\n    # Update our parameters\n    params['max_depth'] = max_depth\n    params['min_child_weight'] = min_child_weight\n    # Run CV\n    cv_results = xgb.cv(\n        params,\n        dtrain,\n        num_boost_round=num_boost_round,\n        seed=42,\n        nfold=5,\n        metrics={'mae'},\n        early_stopping_rounds=10\n    )\n    # Update best MAE\n    mean_mae = cv_results['test-mae-mean'].min()\n    boost_rounds = cv_results['test-mae-mean'].argmin()\n    print(\"\\tMAE {} for {} rounds\".format(mean_mae, boost_rounds))\n    if mean_mae < min_mae:\n        min_mae = mean_mae\n        best_params = (max_depth,min_child_weight)\nprint(\"Best params: {}, {}, MAE: {}\".format(best_params[0], best_params[1], min_mae))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params['max_depth'] = 10\nparams['min_child_weight'] = 6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gridsearch_params = [\n    (subsample, colsample)\n    for subsample in [i/10. for i in range(7,11)]\n    for colsample in [i/10. for i in range(7,11)]\n]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"min_mae = float(\"Inf\")\nbest_params = None\n# We start by the largest values and go down to the smallest\nfor subsample, colsample in reversed(gridsearch_params):\n    print(\"CV with subsample={}, colsample={}\".format(\n                             subsample,\n                             colsample))\n    # We update our parameters\n    params['subsample'] = subsample\n    params['colsample_bytree'] = colsample\n    # Run CV\n    cv_results = xgb.cv(\n        params,\n        dtrain,\n        num_boost_round=num_boost_round,\n        seed=42,\n        nfold=5,\n        metrics={'mae'},\n        early_stopping_rounds=10\n    )\n    # Update best score\n    mean_mae = cv_results['test-mae-mean'].min()\n    boost_rounds = cv_results['test-mae-mean'].argmin()\n    print(\"\\tMAE {} for {} rounds\".format(mean_mae, boost_rounds))\n    if mean_mae < min_mae:\n        min_mae = mean_mae\n        best_params = (subsample,colsample)\nprint(\"Best params: {}, {}, MAE: {}\".format(best_params[0], best_params[1], min_mae))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params['subsample'] = .8\nparams['colsample_bytree'] = 1.","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%time\n# This can take some time…\nmin_mae = float(\"Inf\")\nbest_params = None\nfor eta in [.3, .2, .1, .05, .01, .005]:\n    print(\"CV with eta={}\".format(eta))\n    # We update our parameters\n    params['eta'] = eta\n    # Run and time CV\n    %time cv_results = xgb.cv(params, dtrain,num_boost_round=num_boost_round, seed=42, nfold=5, metrics=['mae'],early_stopping_rounds=10)\n    # Update best score\n    mean_mae = cv_results['test-mae-mean'].min()\n    boost_rounds = cv_results['test-mae-mean'].argmin()\n    print(\"\\tMAE {} for {} rounds\\n\".format(mean_mae, boost_rounds))\n    if mean_mae < min_mae:\n        min_mae = mean_mae\n        best_params = eta\nprint(\"Best params: {}, MAE: {}\".format(best_params, min_mae))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params['eta'] = .01","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params\n{'colsample_bytree': 1.0,\n 'eta': 0.01,\n 'eval_metric': 'mae',\n 'max_depth': 10,\n 'min_child_weight': 6,\n 'objective': 'reg:linear',\n 'subsample': 0.8}","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"model = xgb.train(\n    params,\n    dtrain,\n    num_boost_round=num_boost_round,\n    evals=[(dtest, \"Test\")],\n    early_stopping_rounds=10\n)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"num_boost_round = model.best_iteration + 1\nbest_model = xgb.train(\n    params,\n    dtrain,\n    num_boost_round=num_boost_round,\n    evals=[(dtest, \"Test\")]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean_absolute_error(best_model.predict(dtest), y_test1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model.save_model(\"my_model.model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loaded_model = xgb.Booster()\nloaded_model.load_model(\"my_model.model\")\n# And use it for predictions.\nloaded_model.predict(dtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}