{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport category_encoders as ce \nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.linear_model import Ridge, Lasso, RidgeClassifier\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nimport lightgbm as lgb\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn import metrics\nfrom sklearn.feature_selection import SelectFromModel\nfrom collections import Counter\nimport pickle\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.pipeline import FeatureUnion\nfrom sklearn.impute import MissingIndicator\nfrom sklearn.model_selection import KFold\nfrom sklearn.feature_selection import RFECV\nfrom sklearn.inspection import permutation_importance\nfrom category_encoders import TargetEncoder, LeaveOneOutEncoder\nimport random\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom functools import reduce\nfrom sklearn.linear_model import LogisticRegression\nimport optuna\nfrom optuna.samplers import RandomSampler, GridSampler, TPESampler\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.neural_network import MLPClassifier","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Introduction\n\nThis notebook will show you how to stack your neural network out of sample (oof) outputs with meta features. Three models will be applied to the stacking data and you can combine them at the end: Ridge, Multilayer Perceptron and LightGBM. You can compare this approach with directly using meta in your neural network, or even blend both approaches to get a more robust model. Be aware of data leakage. \n\nInput files: out of sample and test predictions are obtained using Chris Deotte's kernel https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords The files and parameters in this notebook are only for ***demonstration purposes***. You may want to use your own.\n\nYou can certainly apply more trails to find better parameters or experiment with target mean encoding, binning your numeric outputs etc. Please be\naware of overfitting. This notebook gives the opportunities to explore these options. It also allows you to explore different\nimputing strategies: mean, constant etc, as well as different binning strategies, kmeans, uniform, quantile etc. This allows you to\nbuild models based on different datasets to improve robustness.\n\nFocal loss for LightGBM is obtained at: https://github.com/jhwjhw0123/Imbalance-XGBoost/blob/master/imxgboost/focal_loss.py\n\nIf you find any bugs, please let me know. Please upvote if you find this notebook helpful and I really appreciate your support.\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dftrain = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv') \ndftest = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv') \n\noof1 = pd.read_csv('../input/stacking-files/oof1.csv')\nsub1 = pd.read_csv('../input/stacking-files/submission1.csv')\noof1 = oof1[['image_name','pred']]\noof1.columns = ['image_name','pred1']\nsub1.columns = ['image_name','pred1']\n\noof2 = pd.read_csv('../input/stacking-files/oof2.csv')\nsub2 = pd.read_csv('../input/stacking-files/submission2.csv')\noof2 = oof2[['image_name','pred']]\noof2.columns = ['image_name','pred2']\nsub2.columns = ['image_name','pred2']\nprint(oof1.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = dftrain[['image_name','sex','age_approx','anatom_site_general_challenge','target']]\ntest_df = dftest[['image_name','sex','age_approx','anatom_site_general_challenge']]\ntrain_df.columns = ['image_name','sex','age','site','target']\ntest_df.columns = ['image_name','sex','age','site']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_frames = [oof1,oof2]\noof = reduce(lambda  left,right: pd.merge(left,right,on=['image_name'],\n                                            how='outer'), oof_frames)\noof = oof.reset_index(drop=True)\nsub_frames = [sub1,sub2]\nsub = reduce(lambda  left,right: pd.merge(left,right,on=['image_name'],\n                                            how='outer'), sub_frames)\nsub = sub.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pd.merge(oof,train_df,on='image_name',how='outer').reset_index(drop=True)\nX_test = pd.merge(sub,test_df,on='image_name',how='outer').reset_index(drop=True)\nX_train = X_train.dropna(axis=0, how='any', thresh=None, subset=['pred1','pred2'])\ny_train = X_train['target']\nimage_names = X_train['image_name'] \nX_train = X_train[['pred1','pred2','sex','age','site']].reset_index(drop=True)\nX_test = X_test[['pred1','pred2','sex','age','site']].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loss function for LightGBM\ndef focal_binary_lgb(label, pred):\n    def robust_pow(num_base, num_pow):\n        # numpy does not permit negative numbers to fractional power\n        # use this to perform the power algorithmic\n        return np.sign(num_base) * (np.abs(num_base)) ** (num_pow)\n    \n    gamma_indct = 2.0\n    # retrieve data from dtrain matrix\n    #label = dtrain.label\n    # compute the prediction with sigmoid\n    sigmoid_pred = 1.0 / (1.0 + np.exp(-pred))\n    # gradient\n    # complex gradient with different parts\n    g1 = sigmoid_pred * (1 - sigmoid_pred)\n    g2 = label + ((-1) ** label) * sigmoid_pred\n    g3 = sigmoid_pred + label - 1\n    g4 = 1 - label - ((-1) ** label) * sigmoid_pred\n    g5 = label + ((-1) ** label) * sigmoid_pred\n    # combine the gradient\n    grad = gamma_indct * g3 * robust_pow(g2, gamma_indct) * np.log(g4 + 1e-9) + \\\n           ((-1) ** label) * robust_pow(g5, (gamma_indct + 1))\n    # combine the gradient parts to get hessian components\n    hess_1 = robust_pow(g2, gamma_indct) + \\\n             gamma_indct * ((-1) ** label) * g3 * robust_pow(g2, (gamma_indct - 1))\n    hess_2 = ((-1) ** label) * g3 * robust_pow(g2, gamma_indct) / g4\n    # get the final 2nd order derivative\n    hess = ((hess_1 * np.log(g4 + 1e-9) - hess_2) * gamma_indct +\n            (gamma_indct + 1) * robust_pow(g5, gamma_indct)) * g1\n\n    return grad, hess","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define your feature space","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This create the pipeline to create different feature sets for different model to improve the robustness\nof the ensemble","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_Combined(info,model):\n    \n    nontarget_features = info['nontarget_features']\n    target_features = info['target_features']\n    nonbin_features = info['nonbin_features']\n    bin_features = info['bin_features']\n    simple_strategy = info['simple_strategy']\n    kbin_strategy = info['kbin_strategy']\n    \n    if simple_strategy == 'mean':\n        simple_imputer = SimpleImputer(strategy='mean')\n    elif simple_strategy == 'median':\n        simple_imputer = SimpleImputer(strategy='median')\n    else:\n        simple_imputer = SimpleImputer(strategy='constant',fill_value=-1)\n    \n    bin_encode_strategy = kbin_strategy[0]\n    bin_strategy = kbin_strategy[1]\n    if kbin_strategy[2] == 'mean':\n        bin_imputer = SimpleImputer(strategy='mean')\n    else:\n        bin_imputer = SimpleImputer(strategy='median')\n    \n    numeric_transformer1 = FeatureUnion(transformer_list=[\n            ('imputer',simple_imputer),\n            ('indicator',MissingIndicator())\n            ])\n    numeric_transformer2 = Pipeline(steps=[\n            ('imputer', bin_imputer),\n            ('Bin',KBinsDiscretizer(n_bins=5, encode=bin_encode_strategy, strategy=bin_strategy)),\n            ])\n    ce_target = LeaveOneOutEncoder(cols=target_features,sigma=0.1,random_state=0)\n    ce_cat = ce.OneHotEncoder(cols=nontarget_features,handle_unknown='value',handle_missing='value')\n    ct1 = ColumnTransformer(\n            transformers=[\n                ('cat_onehot',ce_cat,nontarget_features),\n                ('cat_target',ce_target,target_features),\n                ('num_nobin',numeric_transformer1,nonbin_features),\n                ('num_bin',numeric_transformer2,bin_features),\n                ],remainder = 'drop')\n    clf_model = Pipeline(steps=[('preprocessor', ct1),\n                          ('classifier', model)])\n    return clf_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I am only using the following feature engineering for **demonstration purposes**.\n\nYou can apply target mean encoding and kmeans discretizer \ndirectly using this code. Play around it with it and see\nwhat features work best with your stacking features\n\nBe careful with overfitting when using target mean encoding\n\nThe order for kbin_strategy is: encode: 'onehot','onehot-dense','ordinal'; strategy: 'uniform', 'quantile', 'kmeans'; I also add an imputer\nto work with KBinDiscretizer. (https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.KBinsDiscretizer.html)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"features_RID = {}\nfeatures_RID['nontarget_features'] = ['site']\nfeatures_RID['target_features'] = ['sex']\nfeatures_RID['nonbin_features'] = ['pred1','pred2','age']\nfeatures_RID['bin_features'] = ['age']\nfeatures_RID['simple_strategy'] = 'mean'\nfeatures_RID['kbin_strategy'] = ['onehot','uniform','mean']\n\nfeatures_LGB = {}\nfeatures_LGB['nontarget_features'] = ['sex','site']\nfeatures_LGB['target_features'] = []\nfeatures_LGB['nonbin_features'] = ['pred1','pred2','age']\nfeatures_LGB['bin_features'] = ['pred1']\nfeatures_LGB['simple_strategy'] = 'constant'\nfeatures_LGB['kbin_strategy'] = ['ordinal','uniform','median']\n\nfeatures_NN = {}\nfeatures_NN['nontarget_features'] = ['sex','site']\nfeatures_NN['target_features'] = []\nfeatures_NN['nonbin_features'] = ['pred1','pred2','age']\nfeatures_NN['bin_features'] = ['pred2']\nfeatures_NN['simple_strategy'] = 'median'\nfeatures_NN['kbin_strategy'] = ['ordinal','uniform','median']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Ridge/MLP/LGB with Optuna","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### TRAIN RIDGE","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name1 = 'Ridge'\nstudy_ridge = optuna.create_study(study_name=study_name1,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_ridge(trial):    \n\n    C = trial.suggest_loguniform('alpha',1e-7,10)\n    b = int(trial.suggest_loguniform('b',1,32))\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    model = RidgeClassifier(alpha=C, class_weight={0:1,1:b},random_state=0)\n    clf = get_model_Combined(features_RID,model)\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        clf, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_ridge.optimize(opt_ridge, n_trials=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_ridge.trials))\ntrial_ridge = study_ridge.best_trial\nprint('Best score : {}'.format(-trial_ridge.value))\nfor key, value in trial_ridge.params.items():\n    print(\"    {}: {}\".format(key, value))\nalpha_RID = list(trial_ridge.params.items())[0][1]\nb = int(list(trial_ridge.params.items())[1][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train LightGBM","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name2 = 'lgb'\nstudy_lgb = optuna.create_study(study_name=study_name2,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_lgb(trial):    \n\n    num_leaves = int(trial.suggest_loguniform(\"num_leaves\", 3,20))\n    subsample =  trial.suggest_discrete_uniform('bfrac',0.5,1.0,q=0.05),\n    subsample_freq = int(trial.suggest_discrete_uniform('bfreq',1,5,q=1.0)),\n    colsample_bytree = trial.suggest_discrete_uniform('feature',0.5,1.0,q=0.05),\n    reg_lambda = trial.suggest_loguniform(\"lambda_l2\", 1e-7, 10)\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    model = lgb.LGBMClassifier(objective=focal_binary_lgb,random_state=0,\n                                                        num_leaves = num_leaves,\n                                                         subsample=subsample,subsample_freq=subsample_freq,\n                                                        colsample_bytree=colsample_bytree,reg_lambda=reg_lambda)\n    clf = get_model_Combined(features_LGB,model)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        clf, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_lgb.optimize(opt_lgb, n_trials=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_lgb.trials))\ntrial_lgb = study_lgb.best_trial\nprint('Best score : {}'.format(-trial_lgb.value))\nfor key, value in trial_lgb.params.items():\n    print(\"    {}: {}\".format(key, value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_leaves = int(list(trial_lgb.params.items())[0][1])\nbfrac = list(trial_lgb.params.items())[1][1]\nbfreq = int(list(trial_lgb.params.items())[2][1])\nfeature =  list(trial_lgb.params.items())[3][1]\nlambda_l2 = list(trial_lgb.params.items())[4][1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train NN","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I only trained 10 trails. You can increase number of trails also\nchange the parameter range values to get better results","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"study_name3 = 'nn'\nstudy_nn = optuna.create_study(study_name=study_name3,direction='maximize',sampler=TPESampler(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def opt_nn(trial):    \n\n    alpha = trial.suggest_loguniform('alpha',1e-6,10)\n    z = int(trial.suggest_loguniform('z',4,32))\n    kFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\n    model= MLPClassifier(hidden_layer_sizes=[z],alpha=alpha,random_state=0,tol=1e-4,max_iter=200)\n    clf = get_model_Combined(features_NN,model)\n    scoring = 'roc_auc'\n    return cross_val_score(\n        clf, X_train, y_train, n_jobs=-1,scoring=scoring,cv=kFold).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_nn.optimize(opt_nn, n_trials=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total number of trials: ',len(study_nn.trials))\ntrial_nn = study_nn.best_trial\nprint('Best score : {}'.format(-trial_nn.value))\nfor key, value in trial_nn.params.items():\n    print(\"    {}: {}\".format(key, value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"alpha_nn = list(trial_nn.params.items())[0][1]\nz = int(list(trial_nn.params.items())[1][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Obtain OOF Score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_final_Ridge = RidgeClassifier(alpha=alpha_RID,class_weight={0:1,1:b},random_state=0)\nmodel_final_LGB = lgb.LGBMClassifier(objective=focal_binary_lgb,random_state=0,\n                                                        num_leaves = num_leaves,\n                                                         subsample=bfrac,subsample_freq=bfreq,\n                                                        colsample_bytree=feature,reg_lambda=lambda_l2)\nmodel_final_NN = MLPClassifier(hidden_layer_sizes=[z],alpha=alpha_nn,random_state=0,tol=1e-4,max_iter=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cv_training(train_data,y_train_data,info,model):\n    kFold= StratifiedKFold(n_splits=5, random_state=10, shuffle=True)\n    oof_pred = []\n    oof_target = []\n    scores = []\n    oof_images = []\n    y_pred_test = 0.0\n    for fold, (trn_idx, val_idx) in enumerate(kFold.split(train_data,y_train_data)):\n        print('Fold: ',fold)\n        X_train_cv = train_data.iloc[trn_idx]\n        X_val_cv = train_data.iloc[val_idx]\n        y_train_cv = y_train_data.iloc[trn_idx]\n        y_val_cv = y_train_data.iloc[val_idx]\n        images_val = image_names.iloc[val_idx]\n        ct = get_model_Combined(info,model)\n        X_train_cv_temp = ct.named_steps['preprocessor'].fit_transform(X_train_cv,y_train_cv)\n        X_val_cv_temp = ct.named_steps['preprocessor'].transform(X_val_cv)\n        X_test_temp = ct.named_steps['preprocessor'].transform(X_test)\n        clf_sigmoid_temp = CalibratedClassifierCV(model, cv=kFold, method='sigmoid')\n        clf_sigmoid_temp.fit(X_train_cv_temp, y_train_cv)\n        y_pred = clf_sigmoid_temp.predict_proba(X_val_cv_temp)[:,1]\n        y_pred_test = y_pred_test + clf_sigmoid_temp.predict_proba(X_test_temp)[:,1]\n        score_temp = roc_auc_score(y_val_cv,y_pred)\n        scores.append(score_temp)\n        oof_pred.append(y_pred)\n        oof_target.append(y_val_cv)\n        oof_images.append(images_val)\n        print(score_temp)\n    y_pred_test = y_pred_test / 5\n    oof_pred = np.concatenate((oof_pred[0],oof_pred[1],oof_pred[2],\n                               oof_pred[3],oof_pred[4]))\n    oof_target = np.concatenate((oof_target[0],oof_target[1],oof_target[2],\n                                 oof_target[3],oof_target[4]))\n    oof_images = np.concatenate((oof_images[0],oof_images[1],oof_images[2],\n                               oof_images[3],oof_images[4]))\n    oof_df = pd.DataFrame({'image_name':oof_images,'predictions':oof_pred,'target':oof_target})\n    return scores, oof_df, y_pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RID_scores,RID_oof_df,y_pred_test_RID = cv_training(X_train,y_train,features_RID,model_final_Ridge)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LGB_scores,LGB_oof_df,y_pred_test_LGB = cv_training(X_train,y_train,features_LGB,model_final_LGB)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NN_scores,NN_oof_df,y_pred_test_NN = cv_training(X_train,y_train,features_NN,model_final_NN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Overall auc RID: ',roc_auc_score(RID_oof_df['target'],RID_oof_df['predictions']))\nprint('Overall auc LGB: ',roc_auc_score(LGB_oof_df['target'],LGB_oof_df['predictions']))\nprint('Overall auc NN: ',roc_auc_score(NN_oof_df['target'],NN_oof_df['predictions']))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RID_oof_df.to_csv('RID_oof_df.csv',index=False)\nLGB_oof_df.to_csv('LGB_oof_df.csv',index=False)\nNN_oof_df.to_csv('NN_oof_df.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Predictions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Do probability calibration for all models\n# https://scikit-learn.org/stable/modules/calibration.html\nkFold= StratifiedKFold(n_splits=5, random_state=0, shuffle=True)\nclf_sigmoid_Ridge = CalibratedClassifierCV(model_final_Ridge, cv=kFold, method=\"sigmoid\")\nclf_sigmoid_LGB = CalibratedClassifierCV(model_final_LGB, cv=kFold, method=\"sigmoid\")\nclf_sigmoid_NN = CalibratedClassifierCV(model_final_NN, cv=kFold, method=\"sigmoid\") ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transform training and testing datasets based on each model's transformer(preprocessor)\nct_RID = get_model_Combined(features_RID,model_final_Ridge)\nX_train_RID = ct_RID.named_steps['preprocessor'].fit_transform(X_train,y_train)\nX_test_RID = ct_RID.named_steps['preprocessor'].transform(X_test)\n\nct_LGB = get_model_Combined(features_LGB,model_final_LGB)\nX_train_LGB = ct_LGB.named_steps['preprocessor'].fit_transform(X_train,y_train)\nX_test_LGB = ct_LGB.named_steps['preprocessor'].transform(X_test)\n\nct_NN = get_model_Combined(features_NN,model_final_NN)\nX_train_NN = ct_NN.named_steps['preprocessor'].fit_transform(X_train,y_train)\nX_test_NN = ct_NN.named_steps['preprocessor'].transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit model to it's own training set\nclf_sigmoid_Ridge.fit(X_train_RID, y_train)\nclf_sigmoid_LGB.fit(X_train_LGB, y_train)\nclf_sigmoid_NN.fit(X_train_NN, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make Predictions\ny_pred_Ridge = clf_sigmoid_Ridge.predict_proba(X_test_RID)[:,1]\ny_pred_LGB = clf_sigmoid_LGB.predict_proba(X_test_LGB)[:,1]\ny_pred_NN = clf_sigmoid_NN.predict_proba(X_test_NN)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here is just a simple average, you can choose your own weights \nsubmission = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsubmission['target'] = 0.4 * y_pred_Ridge + 0.4 * y_pred_LGB + 0.2 * y_pred_NN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If you find this notebook helpful, you may also want to check my other notebooks:\n\nFeature Selection: https://www.kaggle.com/chen2222/feature-selection-mdi-perm-rfe-in-depth-review\n\nLightGBM Tuning:https://www.kaggle.com/chen2222/lightgbm-tuning-step-by-step-optuna-0-122-lb\n\nSimple Blending: https://www.kaggle.com/chen2222/ridge-lgb-nn-on-meta-data-optuna-focal-loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}