{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Those notebooks had really inspired me :<br> \nhttps://www.kaggle.com/code/pourchot/simple-soft-voting<br>\nhttps://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb<br>\nhttps://www.kaggle.com/code/ambrosm/tpsjul22-gaussian-mixture-cluster-analysis<br>\nhttps://www.kaggle.com/code/thedevastator/how-to-ensemble-clustering-algorithms-updated<br>\nhttps://www.kaggle.com/code/eduus710/getting-cluster-ensembles-to-work<br>\nhttps://www.kaggle.com/code/plarmuseau/bruteforce-clustering<br>\nhttps://www.kaggle.com/code/thedevastator/bruteforce-clustering<br>\n(and some others, sorry I don't remember everyone)<br>\nThank you very much.\n\nWith Ricopue's notebook [here][2] I get a 0.61419 score on public leaderboard (3rd place on public leaderboard the 13th july).<br>\nThen I was wondering if I can get a higher score with another method than LGBM.<br>\nI tried a small (and fast) **extratrees on trusted data after BayesianGaussianMixture**, which gave very impressive silhouette / Calinski_Harabasz / Davis_Bouldin scores, but a poor score on public LB.<br>\nAnd then I tried **QDA** was much more impressive. <br>\nAnd then I tried a **soft voting** like in Laurent Pourchot's notebook [here][1].<br>\n\nA high value of AUC score or Accuracy after classification (LGBM or QDA with reg_param == 0) can give a high score on public LeaderBoard. Interesting, isn't ?\n\nMy current best score on public LB is with submission_softvote2.csv.\n\n[1]: https://www.kaggle.com/code/pourchot/simple-soft-voting<br>\n[2]: https://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb<br>","metadata":{}},{"cell_type":"markdown","source":"# Librairies / data ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('max_columns', 100)\npd.set_option('max_rows', 200)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport gc, random, os\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom sklearn.preprocessing import PowerTransformer\n\nfrom sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score\nfrom sklearn.metrics import balanced_accuracy_score, roc_auc_score\n\nfrom sklearn.mixture import BayesianGaussianMixture\n\nimport lightgbm as lgb\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\n\nSEED = 666 # please chose another one than mine\nN_FOLDS = 10\nN_CLUSTERS = 7\n\ndef seed_everything(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:25:21.052810Z","iopub.execute_input":"2022-07-14T04:25:21.053402Z","iopub.status.idle":"2022-07-14T04:25:23.078788Z","shell.execute_reply.started":"2022-07-14T04:25:21.053263Z","shell.execute_reply":"2022-07-14T04:25:23.077586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\", usecols = [f\"f_{i+1:02d}\" for i in range(28)])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:25:23.080932Z","iopub.execute_input":"2022-07-14T04:25:23.081289Z","iopub.status.idle":"2022-07-14T04:25:24.386054Z","shell.execute_reply.started":"2022-07-14T04:25:23.081259Z","shell.execute_reply":"2022-07-14T04:25:24.385205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_scores = []\nusefull_cols = [f\"f_{i:02d}\" for i in list(range(7, 14)) + list(range(22, 29))]\n\ndef scores(preds, lib, df=df[usefull_cols], verbose = True, compute_silhouette = True): \n    \n    # Silhouette is very slow\n    sil = 0\n    if compute_silhouette:\n        sil = silhouette_score(df, preds, metric='euclidean')\n    \n    s = (lib,\n         sil, \n         calinski_harabasz_score(df, preds), \n         davies_bouldin_score(df, preds))\n    \n    if verbose:\n        print(f\"{s[0]} : Silhouette : {s[1]:.1%} | Calinski Harabasz : {s[2]:.1f} | Davis Bouldin : {s[3]:.3f}\")\n        \n    return s","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:25:24.387675Z","iopub.execute_input":"2022-07-14T04:25:24.388356Z","iopub.status.idle":"2022-07-14T04:25:24.405086Z","shell.execute_reply.started":"2022-07-14T04:25:24.388316Z","shell.execute_reply":"2022-07-14T04:25:24.404284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bayesian Gaussian Mixture","metadata":{}},{"cell_type":"code","source":"df_scaled = pd.DataFrame(PowerTransformer().fit_transform(df[usefull_cols]), columns = usefull_cols)\n\nBGM = BayesianGaussianMixture(n_components = N_CLUSTERS, covariance_type = 'full', random_state = SEED, n_init = 5, tol=.01)\nBGM.fit(df_scaled)\n\nBGM_predict = BGM.predict(df_scaled)\nBGM_predict_proba = BGM.predict_proba(df_scaled)\n\nall_scores.append(scores(BGM_predict, lib=\"BayesianGaussianMixture after powertransformer\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:25:24.410128Z","iopub.execute_input":"2022-07-14T04:25:24.410507Z","iopub.status.idle":"2022-07-14T04:29:50.677619Z","shell.execute_reply.started":"2022-07-14T04:25:24.410476Z","shell.execute_reply":"2022-07-14T04:29:50.670053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trusted Data\nhttps://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb<br>\nThanks to Ricopue","metadata":{}},{"cell_type":"code","source":"# get trusted data to train LGB model.\nproba_threshold = .69\n\ndf_scaled['predict'] = BGM_predict\ndf_scaled['predict_proba'] = 0\nfor n in range(N_CLUSTERS):\n    df_scaled[f'predict_proba_{n}'] = BGM_predict_proba[:,n]\n    df_scaled.loc[df_scaled.predict == n, 'predict_proba'] = df_scaled[f'predict_proba_{n}']\n    \n    \nidxs = np.array([])\nfor n in range(N_CLUSTERS):\n    median = df_scaled[df_scaled.predict==n]['predict_proba'].median()\n    idx = df_scaled[(df_scaled.predict==n) & (df_scaled.predict_proba > proba_threshold)].index\n    idxs = np.concatenate((idxs, idx))\n    print(f'Class n°{n}  |  Median : {median:.4f}  |  Training data : {len(idx)/len(df_scaled[(df_scaled.predict==n)]):.1%}')\n    \nX = df_scaled.loc[idxs][usefull_cols]\ny = df_scaled.loc[idxs]['predict']","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:29:50.679878Z","iopub.execute_input":"2022-07-14T04:29:50.680712Z","iopub.status.idle":"2022-07-14T04:29:50.878013Z","shell.execute_reply.started":"2022-07-14T04:29:50.680655Z","shell.execute_reply":"2022-07-14T04:29:50.876990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM on trusted data after BayesianGaussianMixture\nhttps://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb<br>\nThanks to Ricopue","metadata":{}},{"cell_type":"code","source":"params_lgb = {'learning_rate': 0.07,'objective': 'multiclass','boosting': 'gbdt','verbosity': -1,'n_jobs': -1, 'num_classes':N_CLUSTERS} \n\nlgbm_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED)\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X,y)):   \n\n    X_trn = lgb.Dataset(X.iloc[trn_idx], y.iloc[trn_idx], feature_name = usefull_cols)\n    X_val = lgb.Dataset(X.iloc[val_idx], y.iloc[val_idx], feature_name = usefull_cols)\n    \n    model = lgb.train(params = params_lgb, \n                train_set = X_trn, valid_sets =  X_val, \n                num_boost_round = 5000, \n                callbacks = [ lgb.early_stopping(stopping_rounds=100, verbose=True), lgb.log_evaluation(period=200)])  \n    \n    y_pred_proba = model.predict(X.iloc[val_idx])\n    y_pred = np.argmax(y_pred_proba, axis=1)\n    \n    s = (balanced_accuracy_score(y.iloc[val_idx], y_pred),\n        roc_auc_score(y.iloc[val_idx], y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on LGBM. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\\n\")\n    classif_scores.append(s)\n\n    lgbm_predict_proba += model.predict(df_scaled[usefull_cols]) / N_FOLDS\n    \nall_scores.append(scores(np.argmax(lgbm_predict_proba, axis=1), lib=\"LGBM after BayesianGaussianMixture - threshold 0.69\"))\n\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T05:18:09.902334Z","iopub.execute_input":"2022-07-14T05:18:09.903355Z","iopub.status.idle":"2022-07-14T05:27:42.403234Z","shell.execute_reply.started":"2022-07-14T05:18:09.903308Z","shell.execute_reply":"2022-07-14T05:27:42.402219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"High AUC and high Accuracy Score","metadata":{}},{"cell_type":"markdown","source":"# Extratree on trusted data after BayesianGaussianMixture","metadata":{}},{"cell_type":"code","source":"et_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED + 1)\n\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X, y)):   \n\n    X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n    \n    model = ExtraTreesClassifier(n_estimators=100, random_state=SEED)\n    model.fit(X_trn, y_trn)\n    \n    y_pred = model.predict(X_val)\n    y_pred_proba = model.predict_proba(X_val)\n    \n    s = (balanced_accuracy_score(y_val, y_pred),\n        roc_auc_score(y_val, y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on Extratree. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    et_predict_proba += model.predict_proba(df_scaled[usefull_cols]) / N_FOLDS\n\nall_scores.append(scores(np.argmax(et_predict_proba, axis=1), lib=\"Extratree after BayesianGaussianMixture\"))\n\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:57:00.973008Z","iopub.execute_input":"2022-07-14T04:57:00.973422Z","iopub.status.idle":"2022-07-14T04:58:53.057757Z","shell.execute_reply.started":"2022-07-14T04:57:00.973388Z","shell.execute_reply":"2022-07-14T04:58:53.056508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy Score is not so high than with LGBM","metadata":{}},{"cell_type":"markdown","source":"# QuadraticDiscriminantAnalysis on trusted data after BayesianGaussianMixture\nFirst I'm trying to have the better silhouette, ... scores. But AUC and accuracy are very low and it scores only 0.39 on public LB.<br>\nreg_param = 1 : strong regularization per-class covariances.","metadata":{}},{"cell_type":"code","source":"qda_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED + 2)\n\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X, y)):   \n\n    X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n\n    model = QuadraticDiscriminantAnalysis(reg_param=1)\n    model.fit(X_trn, y_trn) # on trusted data only\n    \n    y_pred = model.predict(X_val)\n    y_pred_proba = model.predict_proba(X_val)\n    \n    s = (balanced_accuracy_score(y_val, y_pred),\n        roc_auc_score(y_val, y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on QDA. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    qda_predict_proba += model.predict_proba(df_scaled[usefull_cols]) / N_FOLDS\n\nall_scores.append(scores(np.argmax(qda_predict_proba, axis=1), lib=\"QuadraticDiscriminantAnalysis n°1 after BayesianGaussianMixture\"))\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And I'm trying to have the better AUC and accuracy scores. Score on public LB is higher than 0.615.\nreg_param = 0 : low regularization per-class covariances.","metadata":{}},{"cell_type":"code","source":"qda_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED + 2)\n\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X, y)):   \n\n    X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n\n    model = QuadraticDiscriminantAnalysis(reg_param=0)\n    model.fit(X_trn, y_trn) # on trusted data only\n    \n    y_pred = model.predict(X_val)\n    y_pred_proba = model.predict_proba(X_val)\n    \n    s = (balanced_accuracy_score(y_val, y_pred),\n        roc_auc_score(y_val, y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on QDA. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    qda_predict_proba += model.predict_proba(df_scaled[usefull_cols]) / N_FOLDS\n\nall_scores.append(scores(np.argmax(qda_predict_proba, axis=1), lib=\"QuadraticDiscriminantAnalysis n°2 after BayesianGaussianMixture\"))\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T04:56:10.414586Z","iopub.execute_input":"2022-07-14T04:56:10.414986Z","iopub.status.idle":"2022-07-14T04:56:13.769594Z","shell.execute_reply.started":"2022-07-14T04:56:10.414952Z","shell.execute_reply":"2022-07-14T04:56:13.768465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Better AUC et Accuracy scores than with LGBM. Interesting...","metadata":{}},{"cell_type":"markdown","source":"# Gaussian Naïve Bayes after BayesianGaussianMixture","metadata":{}},{"cell_type":"code","source":"GNB_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED + 2)\n\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X, y)):   \n\n    X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n\n    model = GaussianNB(var_smoothing=.1)\n    model.fit(X_trn, y_trn) # on trusted data only\n    \n    y_pred = model.predict(X_val)\n    y_pred_proba = model.predict_proba(X_val)\n    \n    s = (balanced_accuracy_score(y_val, y_pred),\n        roc_auc_score(y_val, y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on GaussianNB. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    GNB_predict_proba += model.predict_proba(df_scaled[usefull_cols]) / N_FOLDS\n\nall_scores.append(scores(np.argmax(GNB_predict_proba, axis=1), lib=\"GaussianNaïveBayes after BayesianGaussianMixture\"))\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T05:33:50.563717Z","iopub.execute_input":"2022-07-14T05:33:50.564082Z","iopub.status.idle":"2022-07-14T05:35:29.830237Z","shell.execute_reply.started":"2022-07-14T05:33:50.564052Z","shell.execute_reply":"2022-07-14T05:35:29.828919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Linear Discriminant Analysis","metadata":{}},{"cell_type":"code","source":"lda_predict_proba = 0 ; classif_scores = []\n\ngkf = StratifiedKFold(N_FOLDS, shuffle=True, random_state = SEED + 2)\n\nfor fold, (trn_idx, val_idx) in enumerate(gkf.split(X, y)):   \n\n    X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n\n    model = LinearDiscriminantAnalysis()\n    model.fit(X_trn, y_trn) # on trusted data only\n    \n    y_pred = model.predict(X_val)\n    y_pred_proba = model.predict_proba(X_val)\n    \n    s = (balanced_accuracy_score(y_val, y_pred),\n        roc_auc_score(y_val, y_pred_proba, average=\"weighted\", multi_class=\"ovo\"))\n    print(f\"Fold n°{fold+1} on LDA. AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    lda_predict_proba += model.predict_proba(df_scaled[usefull_cols]) / N_FOLDS\n\nall_scores.append(scores(np.argmax(lda_predict_proba, axis=1), lib=\"LinearDiscriminantAnalysis after BayesianGaussianMixture\"))\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T05:05:36.559440Z","iopub.execute_input":"2022-07-14T05:05:36.559994Z","iopub.status.idle":"2022-07-14T05:05:39.882797Z","shell.execute_reply.started":"2022-07-14T05:05:36.559946Z","shell.execute_reply":"2022-07-14T05:05:39.881334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lowest AUC and accuracy scores.","metadata":{}},{"cell_type":"markdown","source":"# Soft voting\nhttps://www.kaggle.com/code/pourchot/simple-soft-voting<br>\nThanks to Laurent Pourchot","metadata":{}},{"cell_type":"code","source":"def soft_voting(preds_probs):\n\n    values = list(range(N_CLUSTERS))\n    pred_test = pd.DataFrame(np.zeros((df.shape[0], 7)), columns = values)\n\n    for i, p in enumerate(preds_probs):\n    \n        MAX = np.argmax(p, axis=1)\n        df[f'pred_{i}'] = MAX\n    \n        # Sort of the prediction by same value of cluster\n        pred_keys = df[f'pred_{i}'].value_counts().index.tolist()\n        pred_dict = dict(zip(pred_keys, values))\n        df[f'pred_{i}'] = df[f'pred_{i}'].map(pred_dict)\n\n        pred_new = pd.DataFrame(p).rename(columns = pred_dict)\n        pred_new = pred_new.reindex(sorted(pred_new.columns), axis=1)\n        pred_test += pred_new # Soft voting by probabiliy addition\n\n    return np.argmax(np.array(pred_test), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T05:33:03.300325Z","iopub.execute_input":"2022-07-14T05:33:03.300701Z","iopub.status.idle":"2022-07-14T05:33:03.310618Z","shell.execute_reply.started":"2022-07-14T05:33:03.300671Z","shell.execute_reply":"2022-07-14T05:33:03.309182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sv1_predict = soft_voting([et_predict_proba, lgbm_predict_proba, qda_predict_proba, lda_predict_proba, GNB_predict_proba])\nall_scores.append(scores(sv1_predict, lib=\"Soft voting n°1 : all\"))\n\nsv2_predict = soft_voting([et_predict_proba, lgbm_predict_proba, qda_predict_proba])\nall_scores.append(scores(sv2_predict, lib=\"Soft voting n°2 : LGBM, extratree and QDA\"))\n\nsv3_predict = soft_voting([lgbm_predict_proba, qda_predict_proba])\nall_scores.append(scores(sv3_predict, lib=\"Soft voting n°3 : LGBM and QDA\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-14T05:35:29.832525Z","iopub.execute_input":"2022-07-14T05:35:29.833344Z","iopub.status.idle":"2022-07-14T05:40:23.876421Z","shell.execute_reply.started":"2022-07-14T05:35:29.833294Z","shell.execute_reply":"2022-07-14T05:40:23.875043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(all_scores, columns=[\"Model\", \"silhouette\", \"Calinski_Harabasz\", \"Davis_Bouldin\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-13T21:05:20.560657Z","iopub.execute_input":"2022-07-13T21:05:20.561724Z","iopub.status.idle":"2022-07-13T21:05:20.584903Z","shell.execute_reply.started":"2022-07-13T21:05:20.561666Z","shell.execute_reply":"2022-07-13T21:05:20.583968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submissions","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\n\nsub['Predicted'] = np.argmax(lgbm_predict_proba, axis = 1)\nsub.to_csv(\"submission_lgbm.csv\",index = False)\n\nsub['Predicted'] = np.argmax(et_predict_proba, axis = 1)\nsub.to_csv(\"submission_extratree.csv\", index = False)\n \nsub['Predicted'] = np.argmax(qda_predict_proba, axis = 1)\nsub.to_csv(\"submission_qda.csv\", index = False)\n \nsub['Predicted'] = np.argmax(lda_predict_proba, axis = 1)\nsub.to_csv(\"submission_lda.csv\", index = False)\n \nsub['Predicted'] = np.argmax(GNB_predict_proba, axis = 1)\nsub.to_csv(\"submission_GNB.csv\", index = False)\n \nsub['Predicted'] = sv1_predict\nsub.to_csv(\"submission_softvote1.csv\", index = False)\n\nsub['Predicted'] = sv2_predict\nsub.to_csv(\"submission_softvote2.csv\", index = False)\n\nsub['Predicted'] = sv3_predict\nsub.to_csv(\"submission_softvote3.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T20:38:16.274548Z","iopub.execute_input":"2022-07-13T20:38:16.274971Z","iopub.status.idle":"2022-07-13T20:38:16.776832Z","shell.execute_reply.started":"2022-07-13T20:38:16.274932Z","shell.execute_reply":"2022-07-13T20:38:16.775708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}