{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello. Beautiful People~!\n  \nI'm not good enough, but I'm going to study the ensemble step by step.    \n  \n**We are only precious beings in the universe.**  \nIf these precious beings can harmonize well, we can do unimaginable things.  \n  \nI believe.  ","metadata":{}},{"cell_type":"markdown","source":"![](https://theclassicalnovice.files.wordpress.com/2015/05/dublin_philharmonic_orchestra_performing_tchaikovskys_symphony_no_4_in_charlotte_north_carolina-e1431911721232.jpg?w=1180&h=435&crop=1)","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-04T07:35:27.735493Z","iopub.execute_input":"2022-03-04T07:35:27.736165Z","iopub.status.idle":"2022-03-04T07:35:27.765809Z","shell.execute_reply.started":"2022-03-04T07:35:27.736047Z","shell.execute_reply":"2022-03-04T07:35:27.764950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ngboost","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-04T07:35:27.767519Z","iopub.execute_input":"2022-03-04T07:35:27.767762Z","iopub.status.idle":"2022-03-04T07:35:42.447077Z","shell.execute_reply.started":"2022-03-04T07:35:27.767730Z","shell.execute_reply":"2022-03-04T07:35:42.446193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# encoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, scale\nfrom sklearn.preprocessing import PolynomialFeatures\n\n# cross validation.. etc\n\nfrom sklearn import model_selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom collections import Counter\n\n# ML\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA, TruncatedSVD\nfrom sklearn import tree\nfrom sklearn import ensemble\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestRegressor\nfrom ngboost import NGBRegressor\nimport xgboost as xgb\n\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\nfrom sklearn.metrics import precision_recall_curve, roc_curve, auc\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nprint(\"Setup Complete\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-04T07:35:42.449004Z","iopub.execute_input":"2022-03-04T07:35:42.449586Z","iopub.status.idle":"2022-03-04T07:35:43.723201Z","shell.execute_reply.started":"2022-03-04T07:35:42.449546Z","shell.execute_reply":"2022-03-04T07:35:43.721996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/petfinder-pawpularity-score'","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.724516Z","iopub.execute_input":"2022-03-04T07:35:43.725255Z","iopub.status.idle":"2022-03-04T07:35:43.728644Z","shell.execute_reply.started":"2022-03-04T07:35:43.725216Z","shell.execute_reply":"2022-03-04T07:35:43.727986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(data_dir + '/train.csv')\ntest = pd.read_csv(data_dir + '/test.csv')\nsample_submission = pd.read_csv(data_dir + '/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.731071Z","iopub.execute_input":"2022-03-04T07:35:43.731490Z","iopub.status.idle":"2022-03-04T07:35:43.782625Z","shell.execute_reply.started":"2022-03-04T07:35:43.731454Z","shell.execute_reply":"2022-03-04T07:35:43.781965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape, sample_submission.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.783727Z","iopub.execute_input":"2022-03-04T07:35:43.784030Z","iopub.status.idle":"2022-03-04T07:35:43.790771Z","shell.execute_reply.started":"2022-03-04T07:35:43.783990Z","shell.execute_reply":"2022-03-04T07:35:43.789977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.792480Z","iopub.execute_input":"2022-03-04T07:35:43.793039Z","iopub.status.idle":"2022-03-04T07:35:43.821456Z","shell.execute_reply.started":"2022-03-04T07:35:43.793003Z","shell.execute_reply":"2022-03-04T07:35:43.820646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.823550Z","iopub.execute_input":"2022-03-04T07:35:43.824163Z","iopub.status.idle":"2022-03-04T07:35:43.841491Z","shell.execute_reply.started":"2022-03-04T07:35:43.824119Z","shell.execute_reply":"2022-03-04T07:35:43.840839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = train.drop(['Pawpularity'], axis=1)\n# df_label = train['Pawpularity'].copy()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.842657Z","iopub.execute_input":"2022-03-04T07:35:43.842900Z","iopub.status.idle":"2022-03-04T07:35:43.846489Z","shell.execute_reply.started":"2022-03-04T07:35:43.842868Z","shell.execute_reply":"2022-03-04T07:35:43.845592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label - feature correlation \n\ndf_matrix = train.corr()\ndf_matrix['Pawpularity'].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.847784Z","iopub.execute_input":"2022-03-04T07:35:43.848214Z","iopub.status.idle":"2022-03-04T07:35:43.869531Z","shell.execute_reply.started":"2022-03-04T07:35:43.848181Z","shell.execute_reply":"2022-03-04T07:35:43.868867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correlation : Group, Accessory, Face / Blur, Subject Focus, Eyes \nattributes = ['Subject Focus', 'Near', 'Action', 'Collage', 'Info', 'Blur', 'Pawpularity'] # i got insight by looking at the scatterplot \nattributes_corr = ['Subject Focus', 'Eyes', 'Face', 'Accessory', 'Group', 'Blur', 'Pawpularity']","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.870730Z","iopub.execute_input":"2022-03-04T07:35:43.870992Z","iopub.status.idle":"2022-03-04T07:35:43.875575Z","shell.execute_reply.started":"2022-03-04T07:35:43.870962Z","shell.execute_reply":"2022-03-04T07:35:43.874740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Use all features","metadata":{}},{"cell_type":"code","source":"# train_label = train['Pawpularity'].copy()\n\nfeatures = [\n    f for f in train.columns if f not in ('Id')\n]\ntrain = train[features]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.877052Z","iopub.execute_input":"2022-03-04T07:35:43.877421Z","iopub.status.idle":"2022-03-04T07:35:43.895210Z","shell.execute_reply.started":"2022-03-04T07:35:43.877384Z","shell.execute_reply":"2022-03-04T07:35:43.894262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label check\nPlease refer to my other kernel : https://www.kaggle.com/kimalpha/petfinder-simple-t-sne-fireworks","metadata":{"execution":{"iopub.status.busy":"2021-11-24T22:51:37.890269Z","iopub.execute_input":"2021-11-24T22:51:37.890937Z","iopub.status.idle":"2021-11-24T22:51:37.904443Z","shell.execute_reply.started":"2021-11-24T22:51:37.890828Z","shell.execute_reply":"2021-11-24T22:51:37.903248Z"}}},{"cell_type":"code","source":"# Ensemble \nfrom functools import partial\nfrom scipy import optimize\n\nclass OptimizeRMSE:\n    def __init__(self):\n        self.coef_ = 0\n        \n    def _rmse(self, coef, X, y):\n        '''\n        :param coef : weight list \n        '''\n        x_coef = X * coef\n        predictions = np.sum(x_coef, axis=1)\n        rmse = np.sqrt(mean_squared_error(y, predictions))\n        \n        return rmse\n    \n    def fit(self, X, y):\n        loss_partial = partial(self._rmse, X=X, y=y)\n        \n        # any distribution where the sum of weights is 1 \n        initial_coef = np.random.dirichlet(np.ones(X.shape[1]), size=1)\n        self.coef_ = optimize.fmin(loss_partial, initial_coef, disp=True)\n    \n    def predict(self, X):\n        x_coef = X * self.coef_\n        predictions = np.sum(x_coef, axis=1)\n        \n        return predictions","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.896852Z","iopub.execute_input":"2022-03-04T07:35:43.897194Z","iopub.status.idle":"2022-03-04T07:35:43.908685Z","shell.execute_reply.started":"2022-03-04T07:35:43.897157Z","shell.execute_reply":"2022-03-04T07:35:43.907898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.iloc[:, :-1].values\ny = train.iloc[:, -1].values\n\nxfold1, xfold2, yfold1, yfold2 = model_selection.train_test_split(\n    X,\n    y,\n    test_size=0.5,\n    stratify=y,\n    shuffle=True,\n    random_state=42\n)\n\npoly2 = PolynomialFeatures(degree=1)\nlr_xfold1 = poly2.fit_transform(xfold1)\n\nlr = LinearRegression()\n\n# Best: {'criterion': 1, 'max_depth': 12.0, 'max_features': 1, 'min_samples_leaf': 1.0, 'min_samples_split': 4.0}\nrf = RandomForestRegressor(random_state=42, n_jobs=-1,\n                           criterion='mae',\n                           max_depth=12,\n                           max_features='sqrt',\n                           min_samples_leaf=1,\n                           min_samples_split=4\n                          )                       \n\nxgbr = xgb.XGBRegressor(random_state=42, \n                        n_jobs=-1,\n                        use_label_encoder=True,\n                        eval_metric='rmse',\n                       )\n\nngb = NGBRegressor()\n\nlr.fit(lr_xfold1, yfold1)\nrf.fit(xfold1, yfold1)\nxgbr.fit(xfold1, yfold1)\nngb.fit(xfold1, yfold1)\n\npred_lr = lr.predict(poly2.fit_transform(xfold2))\npred_rf = rf.predict(xfold2)\npred_xgbr = xgbr.predict(xfold2)\npred_ngb = ngb.predict(xfold2)\navg_pred = (pred_lr + pred_rf + pred_xgbr + pred_ngb) / 4\n\nfold2_preds = np.column_stack((\n    pred_lr,\n    pred_rf,\n    pred_xgbr,\n    pred_ngb,\n    avg_pred\n))\n\nrmse_fold2 = []\n\nfor i in range(fold2_preds.shape[1]):\n    rmse = np.sqrt(mean_squared_error(yfold2, fold2_preds[:, i]))\n    rmse_fold2.append(rmse)\n    \nprint(f\"Linear Regression RMSE : {rmse_fold2[0]}\")\nprint(f\"RandomForest Regressor RMSE : {rmse_fold2[1]}\")\nprint(f\"XGBoost Regressor RMSE: {rmse_fold2[2]}\")\nprint(f\"NGBoost Regressor RMSE: {rmse_fold2[3]}\")\n\nprint(f\"=================================================\")\n# fold change\n\npoly2 = PolynomialFeatures(degree=1)\nlr_xfold2 = poly2.fit_transform(xfold2)\n\nlr = LinearRegression()\n\n# Best: {'criterion': 1, 'max_depth': 12.0, 'max_features': 1, 'min_samples_leaf': 1.0, 'min_samples_split': 4.0}\nrf = RandomForestRegressor(random_state=42, n_jobs=-1,\n                           criterion='mae',\n                           max_depth=12,\n                           max_features='sqrt',\n                           min_samples_leaf=1,\n                           min_samples_split=4\n                          )                       \n\nxgbr = xgb.XGBRegressor(random_state=42, \n                        n_jobs=-1,\n                        use_label_encoder=True,\n                        eval_metric='rmse',\n                       )\n\nngb = NGBRegressor()\n\nlr.fit(lr_xfold2, yfold2)\nrf.fit(xfold2, yfold2)\nxgbr.fit(xfold2, yfold2)\nngb.fit(xfold2, yfold2)\n\npred_lr = lr.predict(poly2.fit_transform(xfold1))\npred_rf = rf.predict(xfold1)\npred_xgbr = xgbr.predict(xfold1)\npred_ngb = ngb.predict(xfold2)\navg_pred = (pred_lr + pred_rf + pred_xgbr + pred_ngb) / 4\n\nfold1_preds = np.column_stack((\n    pred_lr,\n    pred_rf,\n    pred_xgbr,\n    pred_ngb,\n    avg_pred\n))\n\nrmse_fold1 = []\n\nfor i in range(fold1_preds.shape[1]):\n    rmse = np.sqrt(mean_squared_error(yfold1, fold1_preds[:, i]))\n    rmse_fold1.append(rmse)\n    \nprint(f\"Linear Regression RMSE : {rmse_fold1[0]}\")\nprint(f\"RandomForest Regressor RMSE : {rmse_fold1[1]}\")\nprint(f\"XGBoost Regressor RMSE: {rmse_fold1[2]}\")\nprint(f\"NGBoost Regressor RMSE: {rmse_fold1[3]}\")\n\nopt = OptimizeRMSE()\nopt.fit(fold1_preds[:, :-1], yfold1)\nopt_preds_fold2 = opt.predict(fold2_preds[:, :-1])\nrmse = np.sqrt(mean_squared_error(yfold2, opt_preds_fold2))\n\nprint(f\"Optimized RMSE, Fold 2 = {rmse}\")\nprint(f\"Coefficients = {opt.coef_}\")\n\nopt = OptimizeRMSE()\nopt.fit(fold2_preds[:, :-1], yfold2)\nopt_preds_fold1 = opt.predict(fold1_preds[:, :-1])\nrmse = np.sqrt(mean_squared_error(yfold1, opt_preds_fold1))\n\nprint(f\"Optimized RMSE, Fold 1 = {rmse}\")\nprint(f\"Coefficients = {opt.coef_}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:35:43.912395Z","iopub.execute_input":"2022-03-04T07:35:43.912880Z","iopub.status.idle":"2022-03-04T07:36:04.653871Z","shell.execute_reply.started":"2022-03-04T07:35:43.912846Z","shell.execute_reply":"2022-03-04T07:36:04.652733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature selection","metadata":{"execution":{"iopub.status.busy":"2021-11-29T11:37:57.601362Z","iopub.execute_input":"2021-11-29T11:37:57.601819Z","iopub.status.idle":"2021-11-29T11:37:57.607392Z","shell.execute_reply.started":"2021-11-29T11:37:57.601778Z","shell.execute_reply":"2021-11-29T11:37:57.606175Z"}}},{"cell_type":"markdown","source":"* Remove low distributed features. (easy way)  \n* Remove features that have a high correlation. (easy way)      \n* **Univariate Feature Selection: How to prenuclearize features based on each feature's statistical score in the target variable** (mutual information, ANOVA, f-test, chi square)  ","metadata":{}},{"cell_type":"code","source":"X = train.iloc[:, :-1]\ny = train.iloc[:, -1]","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.655285Z","iopub.status.idle":"2022-03-04T07:36:04.656045Z","shell.execute_reply.started":"2022-03-04T07:36:04.655727Z","shell.execute_reply":"2022-03-04T07:36:04.655757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import chi2\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.feature_selection import f_regression\nfrom sklearn.feature_selection import mutual_info_classif\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import SelectPercentile","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.657412Z","iopub.status.idle":"2022-03-04T07:36:04.658056Z","shell.execute_reply.started":"2022-03-04T07:36:04.657803Z","shell.execute_reply":"2022-03-04T07:36:04.657826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UnivariateFeatureSelection:\n    def __init__(self, n_features, problem_type, scoring):\n        if problem_type == \"classification\":\n            valid_scoring = {\n                \"f_classif\": f_classif,\n                \"chi2\": chi2,\n                \"mutual_info_classif\": mutual_info_classif\n            }\n        else:\n            valid_scoring = {\n                \"f_regression\": f_regression,\n                \"mutual_info_regression\": mutual_info_regression\n            }\n        \n        if scoring not in valid_scoring:\n            raise Exception(\"Invalid scoring function, but you are so beautiful :)\")\n        \n        if isinstance(n_features, int):\n            self.selection = SelectKBest(\n                valid_scoring[scoring],\n                k=n_features\n            )\n            \n        elif isinstance(n_features, float):\n            self.selection = SelectPercentile(\n                valid_scoring[scoring],\n                percentile = int(n_features * 100)\n            )\n        \n        else:\n            raise Exception(\"Invalid type of feature\")\n    \n    def fit(self, X, y):\n        return self.selection.fit(X, y)\n    \n    def transform(self, X):\n        return self.selection.transform(X)\n    \n    def fit_transform(self, X, y):\n        return self.selection.fit_transform(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.659238Z","iopub.status.idle":"2022-03-04T07:36:04.659834Z","shell.execute_reply.started":"2022-03-04T07:36:04.659608Z","shell.execute_reply":"2022-03-04T07:36:04.659632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(1) # 12 features, 1 label ","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.661016Z","iopub.status.idle":"2022-03-04T07:36:04.661620Z","shell.execute_reply.started":"2022-03-04T07:36:04.661390Z","shell.execute_reply":"2022-03-04T07:36:04.661415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# what if you want to choose 50 % top features from the classification? \n\nufs = UnivariateFeatureSelection(\n    n_features=0.5,\n    problem_type=\"classification\",\n    scoring='f_classif'\n)\nufs.fit(X, y)\nX_transformed = ufs.transform(X)\nX_transformed.shape # select 12 --> 6 features (50 %)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.662761Z","iopub.status.idle":"2022-03-04T07:36:04.663383Z","shell.execute_reply.started":"2022-03-04T07:36:04.663154Z","shell.execute_reply":"2022-03-04T07:36:04.663177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# what if you want to choose 10 top features from the regression? \n\nufs = UnivariateFeatureSelection(\n    n_features=10,\n    problem_type=\"regression\",\n    scoring='f_regression'\n)\nufs.fit(X, y)\nX_transformed = ufs.transform(X)\nX_transformed.shape # select 12 --> 10 features ","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.664531Z","iopub.status.idle":"2022-03-04T07:36:04.665150Z","shell.execute_reply.started":"2022-03-04T07:36:04.664897Z","shell.execute_reply":"2022-03-04T07:36:04.664932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_x_trans = []\n\nfor n_features in range(1, X.shape[1]):\n    ufs = UnivariateFeatureSelection(\n        n_features=n_features,\n        problem_type=\"regression\",\n        scoring='f_regression'\n    )\n    ufs.fit(X, y)\n    X_transformed = ufs.transform(X)\n    total_x_trans.append(X_transformed)\n\ntotal_x_trans[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.666480Z","iopub.status.idle":"2022-03-04T07:36:04.667094Z","shell.execute_reply.started":"2022-03-04T07:36:04.666837Z","shell.execute_reply":"2022-03-04T07:36:04.666860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I want to find the number of features that minimize the loss function (RMSE) using a single model (NGBoost).  \n  \nso, the univariate feature selection and the greedy feature selection were combined.  \n  \n**@@@Caution@@@**  \nPlease note that the below code **takes a long long time** to execute because the two classes are combined.  \n  \nDepending on the range of choices, Calculate the loss function from 1 to 12 features. ","metadata":{}},{"cell_type":"code","source":"# For the best feature selection, A function that calculates RMSE according to the number of features with a single ml model.(NGBoost)\n\n# Modifying the code.\n\n# class ML_FeatureSelection:\n#     def rmse_score(self, X, y):\n#         model = NGBRegressor() # no hyper-parameter tune \n#         model.fit(X, y)\n#         preds = model.predict(X)\n#         rmse = np.sqrt(mean_squared_error(y, preds))\n        \n#         return rmse\n    \n#     def best_feature_selection(self, X, y):\n#         '''\n#         param X : np array\n#         param y : np array\n#         return : best score, best features\n#         '''\n#         best_features = []\n#         best_scores = []\n        \n#         num_features = X.shape[1]\n        \n#         while True:\n#             first_feature = None\n#             best_score = 0\n            \n#             for feature in range(num_features):\n#                 if feature in best_features:\n#                     continue\n                \n#                 selected_features = best_features + [feature]\n#                 xtrain = X[:, selected_features]\n#                 score = self.rmse_score(xtrain, y)\n                \n#                 if score > best_score:\n#                     first_feature = feature\n#                     best_score = score\n            \n#             if first_feature != None:\n#                 best_features.append(first_feature)\n#                 best_scores.append(best_score)\n            \n#             if len(best_scores) > 2:\n#                 if best_scores[-1] < best_scores[-2]:\n#                     break\n                    \n#         return best_scores[:-1], best_features[:-1]\n    \n#     def __call__(self, X, y):\n#         scores, features = self.best_feature_selection(X, y)\n        \n#         return X[:, features], scores\n\n# if __name__ == '__main__':\n#     for i in range(X.shape[1]):\n#         X = total_x_trans[i]\n#         X_transformed, scores = ML_FeatureSelection()(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:36:04.668283Z","iopub.status.idle":"2022-03-04T07:36:04.668874Z","shell.execute_reply.started":"2022-03-04T07:36:04.668646Z","shell.execute_reply":"2022-03-04T07:36:04.668671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Continue ~","metadata":{}}]}