{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### This is a notebook, which generates predictions for Titanic passenger survival competition.\nIt's accuracy is up to 0.79425, a top 5\\% result.","metadata":{}},{"cell_type":"markdown","source":"### Outline:\n0. Load libraries and custom functions.\n1. Load data.\n2. Preliminary data analysis: explore features and a target, delete unneeded features, create new features.\n3. Train-test split.\n4. Missing values. In some cases it may be useful to explore skew and perform log-transform before imputing missing values.\n5. Feature engineering. Transform skewed variables, do OHC and scaling.\n6. Fit models.\n7. Evaluate models.\n8. Feature importance, error analysis. Based on the results, go to 2. and iterate.\n9. Make predictions.\n\n### To do:\n- Add EDA visualization.\n- Add SHAP feature importances.\n- Add Optuna XGBoost hyperparameter tuning.\n- Add PR curve analysis.","metadata":{}},{"cell_type":"code","source":"# 0. Load libraries #\n\nimport numpy as np\nimport pandas as pd\nimport os, time, warnings, shap, optuna\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer, make_column_transformer\nfrom sklearn.svm import SVC\nfrom sklearn.preprocessing import LabelBinarizer, LabelEncoder, OrdinalEncoder, OneHotEncoder, StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score, cross_val_predict, GridSearchCV, train_test_split, KFold\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, VotingClassifier\nfrom sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score, precision_recall_curve, auc\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, roc_auc_score\nfrom sklearn.inspection import permutation_importance\nfrom xgboost import XGBClassifier\n\npd.set_option('display.max_columns', 20)\npd.set_option('mode.chained_assignment', None)\npd.set_option('display.expand_frame_repr', False)\nwarnings.filterwarnings('ignore')\n\n# Load custom pre-processing functions:\n\ndef draw_histograms(df, variables, n_rows, n_cols):\n    # stolen from https://stackoverflow.com/questions/29530355/plotting-multiple-histograms-in-grid\n    fig=plt.figure()\n    for i, var_name in enumerate(variables):\n        ax=fig.add_subplot(n_rows,n_cols,i+1)\n        df[var_name].hist(bins=10,ax=ax)\n        ax.set_title(var_name+\" Distribution\")\n    fig.tight_layout()  \n    plt.show()\n\n\ndef fillna_mp_i1(df_train, df_test, df_pred, num_features, cat_features, num_fill='median', cat_fill='mode'):\n    \"\"\"This function speeds up filling missing values for 3 main datasets using different imputation methods.\n    Later may replace it with some subclass.\n    Example: fillna_mp_i1(X_train, X_test, X_pred, num_cols, cat_cols)\"\"\"\n    # set df_pred to None if it does not exist\n    if not ((cat_fill=='mode') and (num_fill=='median')):\n        print ('Imputation method not Implemented yet!')\n        return None\n    \n    df_train[num_features] = df_train[num_features].fillna(value=df_train[num_features].median())\n    df_test[num_features] = df_test[num_features].fillna(value=df_train[num_features].median())\n    df_train[cat_features] = df_train[cat_features].fillna(value=df_train[cat_features].mode().iloc[0])\n    df_test[cat_features] = df_test[cat_features].fillna(value=df_train[cat_features].mode().iloc[0])\n    if (df_pred is not None):\n        df_pred[num_features] = df_pred[num_features].fillna(value=df_train[num_features].median())\n        df_pred[cat_features] = df_pred[cat_features].fillna(value=df_train[cat_features].mode().iloc[0])\n    df_train[num_features+cat_features].count\n    \n    all_good = (\n    (np.prod(df_train[num_features+cat_features].shape)==df_train[num_features+cat_features].count().sum()) and \n    (np.prod(df_test[num_features+cat_features].shape) == df_test[num_features+cat_features].count().sum()) and \n    (np.prod(df_pred[num_features+cat_features].shape) == df_pred[num_features+cat_features].count().sum()))\n    if (all_good):\n        print('Missing values imputed successfully')\n    else:\n        print('There are still some missing values...')\n    \ndef add_misDummy_mp_i1(df_train, df_test, df_pred, features):\n    \"\"\"This function creates new dummy columns for missing features.\n    Example: add_misDummy_mp_i1(X_train, X_test, X_pred, ['Age'])\"\"\"\n    # set df_pred to None if it does not exist\n    for feature_name in features:\n        misColName = 'mis'+feature_name\n        df_train.loc[df_train[feature_name].isnull(), misColName]=1\n        df_train.loc[df_train[feature_name].notnull(), misColName]=0\n        df_test.loc[df_test[feature_name].isnull(), misColName]=1\n        df_test.loc[df_test[feature_name].notnull(), misColName]=0\n        if (df_pred is not None):\n            df_pred.loc[df_pred[feature_name].isnull(), misColName]=1\n            df_pred.loc[df_pred[feature_name].notnull(), misColName]=0\n   \n\ndef discretize_mp_i1(df_train, df_test, df_pred, feature, ntiles, delete_feature=False):\n    \"\"\"This function divides a continuous feature into quantile groups.\n    Example: discretize_mp_i1(X_train, X_test, X_pred, 'Age', 15)\"\"\"\n    # set df_pred to None if it does not exist\n    _,bin = pd.qcut(df_train[feature], ntiles, retbins = True, labels = False, duplicates = 'drop')\n    df_train[feature+'Ntile'] = pd.cut(df_train[feature], labels=False, duplicates = 'drop', bins = bin ,include_lowest = True)\n    df_test[feature+'Ntile'] = pd.cut(df_test[feature], labels=False, duplicates = 'drop', bins = bin ,include_lowest = True)\n    if (df_pred is not None):\n        df_pred[feature+'Ntile'] = pd.cut(df_pred[feature], labels=False, duplicates = 'drop', bins = bin ,include_lowest = True)\n    if (delete_feature==True):\n        df_train.drop(columns=[feature], inplace=True)\n        df_test.drop(columns=[feature], inplace=True)\n        df_pred.drop(columns=[feature], inplace=True)\n    print('Discretized ',feature, ' into ', len(bin)-1, ' bins')\n\n\ndef log_transformer_mp_i1(df_train, df_test, df_pred, feature_subset=False, min_skew=3):\n    \"\"\"This function divides a continuous feature into quantile groups.\n    Example: log_transformer_mp_i1(X_train, X_test, X_pred, feature_subset=num_cols)\"\"\"\n    # set df_pred to None if it does not exist\n    if (feature_subset==False):\n        features_totransform = df_train.columns\n    else:\n        features_totransform = feature_subset.copy()\n    skewed_vars = list(df_train.skew()[abs(df_train.skew())>min_skew].index)\n    for col in list(set(skewed_vars)&set(features_totransform)):\n        df_train[col] = np.log1p(df_train[col])\n        df_test[col] = np.log1p(df_test[col])\n        if (df_pred is not None):\n            df_pred[col] = np.log1p(df_pred[col])\n    print('Skewed columns log-transformed: ', list(set(skewed_vars)&set(features_totransform)))\n    \n    \n# 1. Load data #\n\ntime0 = time.time()\n\npath = '../input/titanic/train.csv'\ndf = pd.read_csv(path) \n\ndf.drop(columns=['Name', 'Ticket', 'Cabin', 'PassengerId'],inplace=True)\npred=pd.read_csv('../input/titanic/test.csv')\npred0 = pred.copy()\npred.drop(columns=['Name', 'Ticket', 'Cabin', 'PassengerId'],inplace=True)\n\nprint(df.shape, pred.shape)\n#df.head()\n\n# 2. EDA, adding features #\n\n#df.Survived.value_counts()\ndf['Age2'] = df['Age']**2\npred['Age2'] = pred['Age']**2\n\n# 3. Train-test split #\n\ntrain_y = df[['Survived']]\ntrain_x = df.drop(columns = ['Survived'])\nX_pred = pred.copy()\n\n#bin_cols = [col for col in train_x.columns if train_x[col].nunique()==2]\ncat_cols = [col for col in train_x.columns if train_x[col].nunique() in range(2,10)]\nnum_cols = list(set(train_x.columns)-set(cat_cols))\n\nprint('categorical features: ', cat_cols, 'numerical features: ', num_cols)\n\nX_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size = 0.1, random_state=101)\nprint(X_train.shape, X_test.shape, y_train.shape, X_pred.shape)\n\nX_train.info()\n\n# 4. Misisng values #\n\nadd_misDummy_mp_i1(X_train, X_test, X_pred, ['Age'])\n\nfillna_mp_i1(X_train, X_test, X_pred, num_cols, cat_cols)\n#[X_train.count(), X_test.count(), X_pred.count()]\n\n# extra feature engineering (manual)\n\ndiscretize_mp_i1(X_train, X_test, X_pred, 'Age', 15)\ndiscretize_mp_i1(X_train, X_test, X_pred, 'SibSp', 30)\ndiscretize_mp_i1(X_train, X_test, X_pred, 'Parch', 60)\n\ncat_cols.extend(['misAge', 'AgeNtile', 'SibSpNtile', 'ParchNtile'])\ncat_cols = list(set(cat_cols)-set(['SibSp', 'Parch']))\n\n\n# 5.Feature engineering #\n\nlog_transformer_mp_i1(X_train, X_test, X_pred, feature_subset=num_cols)\n\n# in general, if I plan using raw ols, I should drop one group. o/w, it is beteer to leabe all ohc groups.\n\nfeature_transformer = ColumnTransformer([\n    (\"num\", StandardScaler(), num_cols),\n    (\"cat\", OneHotEncoder(sparse = False, handle_unknown=\"ignore\"), cat_cols),\n    ])\n\nX_train = pd.DataFrame(feature_transformer.fit_transform(X_train), columns=feature_transformer.get_feature_names_out())\nX_test = pd.DataFrame(feature_transformer.transform(X_test), columns=feature_transformer.get_feature_names_out())\nX_pred = pd.DataFrame(feature_transformer.transform(X_pred), columns=feature_transformer.get_feature_names_out())\n\nfewfeatures = ['num__Age', 'num__Age2', 'num__Fare', 'cat__Sex_male', 'cat__Pclass_2', 'cat__Pclass_3']\n\nX_train\n\n# 6. Fit models #\n\nlr = LogisticRegression()\nparam_grid = {'C':[0.3, 1, 3, 10, 30]}\nlrm = GridSearchCV(lr, param_grid, cv=8)\nlrm.fit(X_train, y_train)\nprint('Logistic ', lrm.best_params_, accuracy_score(y_train, lrm.predict(X_train)), roc_auc_score(y_train, lrm.predict(X_train)))\n\nsvm = SVC()\nparam_grid = {'C':[0.3, 1, 2, 3, 10]}\nsvmm = GridSearchCV(svm, param_grid, cv=8)\nsvmm.fit(X_train, y_train)\nprint('SVM ', svmm.best_params_, accuracy_score(y_train, svmm.predict(X_train)), roc_auc_score(y_train, svmm.predict(X_train)))\n\nknn = KNeighborsClassifier()\nparam_grid = dict(n_neighbors=range(2,20))\nknnm = GridSearchCV(knn, param_grid, cv=8)\nknnm.fit(X_train[fewfeatures], y_train)\nprint('KNN ', knnm.best_params_, accuracy_score(y_train, knnm.predict(X_train[fewfeatures])), roc_auc_score(y_train, knnm.predict(X_train[fewfeatures])))\n\ntime1 = time.time()\nrf = RandomForestClassifier()\nparam_grid = {'n_estimators':[100,200], 'max_depth':[2,4,6,8], 'max_features':[4,5,6]}\nrfm = GridSearchCV(rf, param_grid, cv=4)\nrfm.fit(X_train, y_train)\nprint('RF ', rfm.best_params_, accuracy_score(y_train, rfm.predict(X_train)), roc_auc_score(y_train, rfm.predict(X_train)), time.time()-time1)\n\ntime1 = time.time()\nxgb = XGBClassifier()\n# use 'gpu_hist' for more than 10,000 examples.\nparam_grid = {'n_estimators':[200], 'max_depth':[3,4], 'eta':[0.03, 0.04, 0.05], 'subsample':[0.6, 0.8],\n             'colsample_bytree':[0.4, 0.6]}\nxgbm = GridSearchCV(xgb, param_grid, cv=2)\nxgbm.fit(X_train, y_train)\nprint('XGB ', xgbm.best_params_, accuracy_score(y_train, xgbm.predict(X_train)), roc_auc_score(y_train, xgbm.predict(X_train)), time.time()-time1)\nprint('XGB', f1_score(y_train,xgbm.predict(X_train)), recall_score(y_train,xgbm.predict(X_train)), precision_score(y_train,xgbm.predict(X_train)))\n\n# 7. accuracy #\n\nprint('Out of Sample:')\nprint('Logistic ', accuracy_score(y_test, lrm.predict(X_test)), roc_auc_score(y_test, lrm.predict(X_test)))\nprint('SVM ', accuracy_score(y_test, svmm.predict(X_test)), roc_auc_score(y_test, svmm.predict(X_test)))\nprint('KNN ', accuracy_score(y_test, knnm.predict(X_test[fewfeatures])), roc_auc_score(y_test, knnm.predict(X_test[fewfeatures])))\nprint('RF ', accuracy_score(y_test, rfm.predict(X_test)), roc_auc_score(y_test, rfm.predict(X_test)))\nprint('XGB ', accuracy_score(y_test, xgbm.predict(X_test)), roc_auc_score(y_test, xgbm.predict(X_test)))\nprint('Total time ', time.time()-time0)\n\n# VotingClassifier:\n\nestimator = []\n#estimator.append(('LR', LogisticRegression(C=1)))\nestimator.append(('SVM', SVC(C=1, probability = True)))\n#estimator.append(('KNN', KNeighborsClassifier(n_neighbors=5)))\nestimator.append(('RF', RandomForestClassifier(max_depth=5, max_features=4, n_estimators=200)))\nestimator.append(('XGB', XGBClassifier(eta=0.04, max_depth=3, n_estimators=200, \n                                       subsample=0.6, colsample_bytree=0.6)))\nvot_soft = VotingClassifier(estimators = estimator, voting ='soft')\nvot_soft.fit(X_train, y_train)\nprint('VotingClassifier5 ', accuracy_score(y_train, vot_soft.predict(X_train)), roc_auc_score(y_train, vot_soft.predict(X_train)))\nprint('VotingClassifier5 ', accuracy_score(y_test, vot_soft.predict(X_test)), roc_auc_score(y_test, vot_soft.predict(X_test)))\n# to add KNN with different feature sets, \n# see https://stackoverflow.com/questions/45074579/votingclassifier-different-feature-sets","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:27:47.786017Z","iopub.execute_input":"2022-08-06T03:27:47.786477Z","iopub.status.idle":"2022-08-06T03:28:58.199080Z","shell.execute_reply.started":"2022-08-06T03:27:47.786408Z","shell.execute_reply":"2022-08-06T03:28:58.198081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optuna XGB hyperparameter optimization\n\ntime1=time.time()\n\n### Fit XGBoost using Optuna hyperparameter optimization ###\n\ndef objective(trial, cv_runs=1, n_splits=2, n_jobs=-1, scale_pos_weight=1, early_stopping_rounds=50):\n\n    cv_regularizer=0.05\n    # Usually values between 0.1 and 0.2 work fine.\n\n    params = {\n        #\"tree_method\": 'gpu_hist',\n        \"verbosity\": 0,  # 0 (silent) - 3 (debug)\n        #\"n_estimators\": trial.suggest_int(\"n_estimators\", 500, 1000),\n        \"n_estimators\": 500,\n        \"max_depth\": trial.suggest_int(\"max_depth\", 2, 10),\n        \"learning_rate\": trial.suggest_uniform(\"learning_rate\", 0.01, 0.2),\n        \"colsample_bytree\": trial.suggest_uniform(\"colsample_bytree\", 0.1, 0.95),\n        \"subsample\": trial.suggest_uniform(\"subsample\", 0.5, 0.95),\n        \"alpha\": trial.suggest_loguniform(\"alpha\", 0.1, 10.0),\n        \"lambda\": trial.suggest_loguniform(\"lambda\", 0.1, 150.0),\n        \"gamma\": trial.suggest_loguniform(\"gamma\", 1e-10, 10.0),\n        \"min_child_weight\": trial.suggest_loguniform(\"min_child_weight\", 0.1, 10),\n        \"n_jobs\": n_jobs,\n    }\n    # usually it makes sense to resrtict hyperparameter space from some solutions which Optuna will find\n    # e.g., for tmx-joined data only (downsampled tmx), optuna keeps selecting depths of 2 and 3.\n    # for my purposes (smooth left side of prc, close to 1), those solutions are no good.\n\n    temp_out = []\n\n    for i in range(cv_runs):\n\n        X = X_train\n        y = y_train.Survived\n\n        model = XGBClassifier(**params)\n        rkf = KFold(n_splits=n_splits, shuffle=True)\n        X_values = X.values\n        y_values = y.values\n        y_pred = np.zeros_like(y_values)\n        y_pred_train = np.zeros_like(y_values)\n        for train_index, test_index in rkf.split(X_values):\n            X_A, X_B = X_values[train_index, :], X_values[test_index, :]\n            y_A, y_B = y_values[train_index], y_values[test_index]\n            model.fit(X_A, y_A, eval_set=[(X_B, y_B)],\n                      early_stopping_rounds=early_stopping_rounds, verbose = False)\n            y_pred[test_index] += model.predict(X_B)\n            y_pred_train[train_index] += model.predict(X_A)\n        score_train = roc_auc_score(y_train, y_pred_train)\n        score_test = roc_auc_score(y_train, y_pred) \n        overfit = score_train-score_test\n        #return (score_test)\n        #return (score_test-cv_regularizer*overfit)\n        temp_out.append(score_test-cv_regularizer*overfit)\n\n    return (np.mean(temp_out))\n\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=40)\nprint('Total time for hypermarameter optimization ', time.time()-time1)\nhp = study.best_params\nfor key, value in hp.items():\n    print(f\"{key:>20s} : {value}\")\nprint(f\"{'best objective value':>20s} : {study.best_value}\")\n\noptuna_hyperpars = study.best_params\n#optuna_hyperpars['tree_method']='gpu_hist'\noptuna_hyperpars['scale_pos_weight']=1\n#optuna_hyperpars['early_stopping_rounds']=50\n\noptuna_xgb = XGBClassifier(**optuna_hyperpars)\noptuna_xgb.fit(X_train, y_train)\n\ndisplay('Accuracy: ', accuracy_score(y_train,optuna_xgb.predict(X_train)))\ndisplay('F1 score: ', f1_score(y_train,optuna_xgb.predict(X_train)))\ndisplay('Recall score: ', recall_score(y_train,optuna_xgb.predict(X_train)))\ndisplay('Precision score: ', precision_score(y_train,optuna_xgb.predict(X_train)))\ndisplay('ROCAUC: ', roc_auc_score(y_train,optuna_xgb.predict(X_train)))\n#display('Precision at 20% recall: ', r20prec_train)\n#display('Precision at 50% recall: ', r50prec_train)\n# Performance evaluation:\ndisplay('Accuracy: ', accuracy_score(y_test,optuna_xgb.predict(X_test)))\ndisplay('F1 score: ', f1_score(y_test,optuna_xgb.predict(X_test)))\ndisplay('Recall score: ', recall_score(y_test,optuna_xgb.predict(X_test)))\ndisplay('Precision score: ', precision_score(y_test,optuna_xgb.predict(X_test)))\ndisplay('ROCAUC: ', roc_auc_score(y_test,optuna_xgb.predict(X_test)))\n#display('Precision at 20% recall: ', r20prec_test)\n#display('Precision at 50% recall: ', r50prec_test)\ndisplay(time.time()-time0)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:28:58.201089Z","iopub.execute_input":"2022-08-06T03:28:58.201349Z","iopub.status.idle":"2022-08-06T03:31:41.082390Z","shell.execute_reply.started":"2022-08-06T03:28:58.201317Z","shell.execute_reply":"2022-08-06T03:31:41.081506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8. feature importance #\n\nresults = permutation_importance(xgbm, X_test, y_test, scoring='accuracy', n_jobs=-1)\nfi_lr = pd.DataFrame({'col':X_test.columns, 'FI':results.importances_mean})\n#fi_lr.sort_values('FI', ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:31:41.083704Z","iopub.execute_input":"2022-08-06T03:31:41.083935Z","iopub.status.idle":"2022-08-06T03:31:44.069976Z","shell.execute_reply.started":"2022-08-06T03:31:41.083908Z","shell.execute_reply":"2022-08-06T03:31:44.068888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 9. predictions #\n\nsubmission_df_vc = pd.DataFrame({'PassengerId': pred0.PassengerId, 'Survived': vot_soft.predict(X_pred)}, columns=['PassengerId', 'Survived'])\nsubmission_df_svm = pd.DataFrame({'PassengerId': pred0.PassengerId, 'Survived': svmm.predict(X_pred)}, columns=['PassengerId', 'Survived'])\nsubmission_df_rf = pd.DataFrame({'PassengerId': pred0.PassengerId, 'Survived': rfm.predict(X_pred)}, columns=['PassengerId', 'Survived'])\nsubmission_df_bt = pd.DataFrame({'PassengerId': pred0.PassengerId, 'Survived': xgbm.predict(X_pred)}, columns=['PassengerId', 'Survived'])\nsubmission_df_oxgb = pd.DataFrame({'PassengerId': pred0.PassengerId, 'Survived': optuna_xgb.predict(X_pred)}, columns=['PassengerId', 'Survived'])\n\nsubmission_df_vc.to_csv('KP10_v2_vc.csv',index=False)\nsubmission_df_svm.to_csv('KP10_v2_svm.csv',index=False)\nsubmission_df_rf.to_csv('KP10_v2_rf.csv',index=False)\nsubmission_df_bt.to_csv('KP10_v2_bt.csv',index=False)\nsubmission_df_oxgb.to_csv('KP10_oxgb.csv',index=False)\n\nos.chdir(r'/kaggle/working')\n\nfrom IPython.display import FileLink\nFileLink(r'KP10_oxgb.csv')\n\n#display(time.time()-time0)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:32:53.647325Z","iopub.execute_input":"2022-08-06T03:32:53.647730Z","iopub.status.idle":"2022-08-06T03:32:53.795134Z","shell.execute_reply.started":"2022-08-06T03:32:53.647692Z","shell.execute_reply":"2022-08-06T03:32:53.794040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Appendix:\n#### Exploring XGBoost predictions with PR curve","metadata":{}},{"cell_type":"code","source":"# Precision-Recall curve in train set\nprecision_t, recall_t, threshold = precision_recall_curve(y_train, xgbm.predict_proba(X_train)[:, 1])\nauc_precision_recall_train = auc(recall_t, precision_t)\ntemp = recall_t[(recall_t>0.195)&(recall_t<0.205)]\ntemp = temp[int(len(temp)/2)]\nindexx = ((np.where(recall_t==temp)))[0][0]\nr20prec_train = precision_t[indexx]\n\nfig, ax = plt.subplots()\nax.plot(recall_t, precision_t, color='purple')\nax.set_title('Precision-Recall Curve, train set')\nax.set_ylabel('Precision')\nax.set_xlabel('Recall')\nax.set_ylim(bottom=0, top=1.02)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:31:44.230537Z","iopub.execute_input":"2022-08-06T03:31:44.231342Z","iopub.status.idle":"2022-08-06T03:31:44.446989Z","shell.execute_reply.started":"2022-08-06T03:31:44.231300Z","shell.execute_reply":"2022-08-06T03:31:44.446095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(xgbm.predict_proba(X_train)[:, 1])\ndisplay(precision_t, recall_t, threshold)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:31:44.448443Z","iopub.execute_input":"2022-08-06T03:31:44.448926Z","iopub.status.idle":"2022-08-06T03:31:44.495171Z","shell.execute_reply.started":"2022-08-06T03:31:44.448888Z","shell.execute_reply":"2022-08-06T03:31:44.494251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Precision-Recall curve in test set\nprecision_t, recall_t, threshold = precision_recall_curve(y_test, xgbm.predict_proba(X_test)[:, 1])\nauc_precision_recall_train = auc(recall_t, precision_t)\ntemp = recall_t[(recall_t>0.18)&(recall_t<0.22)]\ntemp = temp[int(len(temp)/2)]\nindexx = ((np.where(recall_t==temp)))[0][0]\nr20prec_train = precision_t[indexx]\n\nfig, ax = plt.subplots()\nax.plot(recall_t, precision_t, color='purple')\nax.set_title('Precision-Recall Curve, test set')\nax.set_ylabel('Precision')\nax.set_xlabel('Recall')\nax.set_ylim(bottom=0, top=1.02)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:31:44.496531Z","iopub.execute_input":"2022-08-06T03:31:44.496793Z","iopub.status.idle":"2022-08-06T03:31:44.707358Z","shell.execute_reply.started":"2022-08-06T03:31:44.496759Z","shell.execute_reply":"2022-08-06T03:31:44.706484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for those examples, which the model is confident about, it could make sense to set predicted probability as 1.","metadata":{"execution":{"iopub.status.busy":"2022-08-06T03:31:44.708969Z","iopub.execute_input":"2022-08-06T03:31:44.710619Z","iopub.status.idle":"2022-08-06T03:31:44.717433Z","shell.execute_reply.started":"2022-08-06T03:31:44.710510Z","shell.execute_reply":"2022-08-06T03:31:44.715207Z"},"trusted":true},"execution_count":null,"outputs":[]}]}