{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install autoviz;","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom scipy.stats import norm, skew\nfrom autoviz.AutoViz_Class import AutoViz_Class\nimport warnings\n\nwarnings.filterwarnings('ignore')\n%config InlineBackend.figure_format = 'svg' \n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:28:45.071954Z","iopub.execute_input":"2022-07-10T12:28:45.0724Z","iopub.status.idle":"2022-07-10T12:28:50.398324Z","shell.execute_reply.started":"2022-07-10T12:28:45.072363Z","shell.execute_reply":"2022-07-10T12:28:50.39725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.cluster import KMeans, AgglomerativeClustering\nfrom sklearn.linear_model import LinearRegression, LassoCV, RidgeCV\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.model_selection import (train_test_split, cross_val_score,\n                                    KFold, GridSearchCV)\nfrom scipy import stats\nfrom scipy.special import boxcox1p\nfrom sklearn.preprocessing import LabelEncoder, RobustScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.model_selection import (train_test_split, cross_val_score,\n                                    KFold, GridSearchCV)\n\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\nfrom sklearn.kernel_ridge import KernelRidge\n\nimport xgboost as xgb\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:28:54.756943Z","iopub.execute_input":"2022-07-10T12:28:54.757348Z","iopub.status.idle":"2022-07-10T12:28:55.87593Z","shell.execute_reply.started":"2022-07-10T12:28:54.757314Z","shell.execute_reply":"2022-07-10T12:28:55.874933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\nvalid = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\n\ntrain_ID = train['Id']\nvalid_ID = valid['Id']\n\ntrain.drop(\"Id\", axis = 1, inplace = True)\nvalid.drop(\"Id\", axis = 1, inplace = True)\n\nprint('Train:{}   Valid:{}'.format(train.shape, valid.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:29:17.379138Z","iopub.execute_input":"2022-07-10T12:29:17.379525Z","iopub.status.idle":"2022-07-10T12:29:17.465274Z","shell.execute_reply.started":"2022-07-10T12:29:17.379493Z","shell.execute_reply":"2022-07-10T12:29:17.464042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA & VISUALIZATION","metadata":{}},{"cell_type":"code","source":"AV = AutoViz_Class()\ndf_av = AV.AutoViz('../input/house-prices-advanced-regression-techniques/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:29:43.663758Z","iopub.execute_input":"2022-07-10T12:29:43.664169Z","iopub.status.idle":"2022-07-10T12:30:42.841358Z","shell.execute_reply.started":"2022-07-10T12:29:43.664139Z","shell.execute_reply":"2022-07-10T12:30:42.839974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, ax = plt.subplots(1,3, figsize = (20,8))\nsns.stripplot(data=train, x = 'OverallQual', y='SalePrice', ax = ax[0])\nsns.violinplot(data=train, x = 'OverallQual', y='SalePrice', ax = ax[1])\nsns.boxplot(data=train, x = 'OverallQual', y='SalePrice', ax = ax[2])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:31:09.387958Z","iopub.execute_input":"2022-07-10T12:31:09.388464Z","iopub.status.idle":"2022-07-10T12:31:10.395072Z","shell.execute_reply.started":"2022-07-10T12:31:09.388421Z","shell.execute_reply":"2022-07-10T12:31:10.393357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'YearBuilt'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\nf, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:31:23.790667Z","iopub.execute_input":"2022-07-10T12:31:23.791054Z","iopub.status.idle":"2022-07-10T12:31:28.406222Z","shell.execute_reply.started":"2022-07-10T12:31:23.791023Z","shell.execute_reply":"2022-07-10T12:31:28.404913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Log-transformation of the variables","metadata":{}},{"cell_type":"code","source":"sns.distplot(train['SalePrice'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:31:43.930774Z","iopub.execute_input":"2022-07-10T12:31:43.931172Z","iopub.status.idle":"2022-07-10T12:31:44.596661Z","shell.execute_reply.started":"2022-07-10T12:31:43.931143Z","shell.execute_reply":"2022-07-10T12:31:44.595308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Skewness: %f\" % train['SalePrice'].skew())\nprint(\"Kurtosis: %f\" % train['SalePrice'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:31:56.29973Z","iopub.execute_input":"2022-07-10T12:31:56.300174Z","iopub.status.idle":"2022-07-10T12:31:56.307101Z","shell.execute_reply.started":"2022-07-10T12:31:56.300143Z","shell.execute_reply":"2022-07-10T12:31:56.30577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['SalePrice'] = np.log1p(train[\"SalePrice\"])\nsns.distplot(train['SalePrice'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['SalePrice'], plot=plt)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:31:58.340272Z","iopub.execute_input":"2022-07-10T12:31:58.340662Z","iopub.status.idle":"2022-07-10T12:31:58.854608Z","shell.execute_reply.started":"2022-07-10T12:31:58.34063Z","shell.execute_reply":"2022-07-10T12:31:58.853442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Skewness: %f\" % train['SalePrice'].skew())\nprint(\"Kurtosis: %f\" % train['SalePrice'].kurt())","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:01.884953Z","iopub.execute_input":"2022-07-10T12:32:01.885335Z","iopub.status.idle":"2022-07-10T12:32:01.893246Z","shell.execute_reply.started":"2022-07-10T12:32:01.885304Z","shell.execute_reply":"2022-07-10T12:32:01.892218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering ","metadata":{}},{"cell_type":"code","source":"ntrain = train.shape[0]\nnvalid = valid.shape[0]\ny_train = train.SalePrice.values\nall_data = pd.concat((train, valid)).reset_index(drop=True)\nall_data.drop(['SalePrice'], axis=1, inplace=True)\nprint(\"all_data size is : {}\".format(all_data.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:04.499885Z","iopub.execute_input":"2022-07-10T12:32:04.500375Z","iopub.status.idle":"2022-07-10T12:32:04.538251Z","shell.execute_reply.started":"2022-07-10T12:32:04.500335Z","shell.execute_reply":"2022-07-10T12:32:04.537223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)[:7]\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head(7)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:10.598314Z","iopub.execute_input":"2022-07-10T12:32:10.598694Z","iopub.status.idle":"2022-07-10T12:32:10.628819Z","shell.execute_reply.started":"2022-07-10T12:32:10.598662Z","shell.execute_reply":"2022-07-10T12:32:10.627675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the data description, it said that for certain variables, NA means that the house has no some  specifications.For example, FireplaceQu:  NA means house don't have fireplace. So we need to replace Na with None to don't lose descriptive power.","metadata":{}},{"cell_type":"code","source":"none_list = ['PoolQC','MiscFeature','Alley','Fence','FireplaceQu',\n             'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',\n             'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',\n             'BsmtFinType2','MasVnrType','MSSubClass']\n\nzero_list = ['GarageYrBlt', 'GarageArea', 'GarageCars',\n            'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',\n            'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath',\n            'MasVnrArea']\n\ndrop_list = ['Utilities']\n\nfor el in none_list:\n    all_data[el] = all_data[el].fillna(\"None\")\n\nfor el in zero_list:\n    all_data[el] = all_data[el].fillna(0)\n\nall_data[\"LotFrontage\"] = all_data.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(\n    lambda x: x.fillna(x.median()))\n\nall_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode()[0])\nall_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode()[0])\nall_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode()[0])\nall_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode()[0])\nall_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode()[0])\nall_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode()[0])\nall_data[\"Functional\"] = all_data[\"Functional\"].fillna(\"Typ\")\n\nall_data = all_data.drop(drop_list, axis=1)\n\nall_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:12.538817Z","iopub.execute_input":"2022-07-10T12:32:12.53923Z","iopub.status.idle":"2022-07-10T12:32:12.619739Z","shell.execute_reply.started":"2022-07-10T12:32:12.539198Z","shell.execute_reply":"2022-07-10T12:32:12.618608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']\nall_data[\"SqFtPerRoom\"] = all_data[\"GrLivArea\"] / (all_data[\"TotRmsAbvGrd\"] +\n                                                       all_data[\"FullBath\"] +\n                                                       all_data[\"HalfBath\"] +\n                                                       all_data[\"KitchenAbvGr\"])\n\nall_data['Total_Home_Quality'] = all_data['OverallQual'] + all_data['OverallCond']\n\nall_data['Total_Bathrooms'] = (all_data['FullBath'] + (0.5 * all_data['HalfBath']) +\n                               all_data['BsmtFullBath'] + (0.5 * all_data['BsmtHalfBath']))\n\nall_data[\"HighQualSF\"] = all_data[\"GrLivArea\"]+all_data[\"1stFlrSF\"] + all_data[\"2ndFlrSF\"]\n+0.5*all_data[\"GarageArea\"]+0.5*all_data[\"TotalBsmtSF\"]+1*all_data[\"MasVnrArea\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:13.466559Z","iopub.execute_input":"2022-07-10T12:32:13.466966Z","iopub.status.idle":"2022-07-10T12:32:13.490688Z","shell.execute_reply.started":"2022-07-10T12:32:13.46693Z","shell.execute_reply":"2022-07-10T12:32:13.489846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)\nall_data['OverallCond'] = all_data['OverallCond'].astype(str)\nall_data['YrSold'] = all_data['YrSold'].astype(str)\nall_data['MoSold'] = all_data['MoSold'].astype(str)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:14.388044Z","iopub.execute_input":"2022-07-10T12:32:14.388431Z","iopub.status.idle":"2022-07-10T12:32:14.407924Z","shell.execute_reply.started":"2022-07-10T12:32:14.388402Z","shell.execute_reply":"2022-07-10T12:32:14.406851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\ncols = ('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', \n        'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1', \n        'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',\n        'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond', \n        'YrSold', 'MoSold')\n# process columns, apply LabelEncoder to categorical features\nfor c in cols:\n    lbl = LabelEncoder() \n    lbl.fit(list(all_data[c].values)) \n    all_data[c] = lbl.transform(list(all_data[c].values))\n\n# shape        \nprint('Shape all_data: {}'.format(all_data.shape))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:15.002704Z","iopub.execute_input":"2022-07-10T12:32:15.003446Z","iopub.status.idle":"2022-07-10T12:32:15.150314Z","shell.execute_reply.started":"2022-07-10T12:32:15.003402Z","shell.execute_reply":"2022-07-10T12:32:15.149186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n\n# Check the skew of all numerical features\nskewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)\nprint(\"\\nSkew in numerical features: \\n\")\nskewness = pd.DataFrame({'Skew' :skewed_feats})\nskewness.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:16.087496Z","iopub.execute_input":"2022-07-10T12:32:16.087897Z","iopub.status.idle":"2022-07-10T12:32:16.127199Z","shell.execute_reply.started":"2022-07-10T12:32:16.087863Z","shell.execute_reply":"2022-07-10T12:32:16.126372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kewness = skewness[abs(skewness) > 0.75]\nprint(\"There are {} skewed numerical features to Box Cox transform\".format(skewness.shape[0]))\n\nskewed_features = skewness.index\nlam = 0.15\nfor feat in skewed_features:\n    #all_data[feat] += 1\n    all_data[feat] = boxcox1p(all_data[feat], lam)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:17.612042Z","iopub.execute_input":"2022-07-10T12:32:17.613089Z","iopub.status.idle":"2022-07-10T12:32:17.654224Z","shell.execute_reply.started":"2022-07-10T12:32:17.613051Z","shell.execute_reply":"2022-07-10T12:32:17.652999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.get_dummies(all_data)\nprint(all_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:18.260714Z","iopub.execute_input":"2022-07-10T12:32:18.261719Z","iopub.status.idle":"2022-07-10T12:32:18.296778Z","shell.execute_reply.started":"2022-07-10T12:32:18.261675Z","shell.execute_reply":"2022-07-10T12:32:18.295304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = all_data[:ntrain]\nvalid_df = all_data[ntrain:]","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:18.999834Z","iopub.execute_input":"2022-07-10T12:32:19.000225Z","iopub.status.idle":"2022-07-10T12:32:19.005228Z","shell.execute_reply.started":"2022-07-10T12:32:19.00019Z","shell.execute_reply":"2022-07-10T12:32:19.00409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"n_folds = 5\n\ndef rmsle_cv(model):\n    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(train_df.values)\n    rmse= np.sqrt(-cross_val_score(model, train_df.values, y_train, scoring=\"neg_mean_squared_error\", cv = kf))\n    return(rmse)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:19.739978Z","iopub.execute_input":"2022-07-10T12:32:19.740885Z","iopub.status.idle":"2022-07-10T12:32:19.746275Z","shell.execute_reply.started":"2022-07-10T12:32:19.740848Z","shell.execute_reply":"2022-07-10T12:32:19.745414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmsle(y, y_pred):\n    return np.sqrt(mean_squared_error(y, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:20.15728Z","iopub.execute_input":"2022-07-10T12:32:20.157674Z","iopub.status.idle":"2022-07-10T12:32:20.163469Z","shell.execute_reply.started":"2022-07-10T12:32:20.15764Z","shell.execute_reply":"2022-07-10T12:32:20.16195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=42))\nscore = rmsle_cv(lasso)\nprint(\"\\nLasso score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:20.688487Z","iopub.execute_input":"2022-07-10T12:32:20.689246Z","iopub.status.idle":"2022-07-10T12:32:22.198889Z","shell.execute_reply.started":"2022-07-10T12:32:20.689191Z","shell.execute_reply":"2022-07-10T12:32:22.197455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=42))\nscore = rmsle_cv(ENet)\nprint(\"ElasticNet score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:22.204235Z","iopub.execute_input":"2022-07-10T12:32:22.205207Z","iopub.status.idle":"2022-07-10T12:32:23.87268Z","shell.execute_reply.started":"2022-07-10T12:32:22.205152Z","shell.execute_reply":"2022-07-10T12:32:23.871177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)\nscore = rmsle_cv(KRR)\nprint(\"Kernel Ridge score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:23.875216Z","iopub.execute_input":"2022-07-10T12:32:23.876115Z","iopub.status.idle":"2022-07-10T12:32:24.371858Z","shell.execute_reply.started":"2022-07-10T12:32:23.876063Z","shell.execute_reply":"2022-07-10T12:32:24.370546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,\n                                   max_depth=4, max_features='sqrt',\n                                   min_samples_leaf=15, min_samples_split=10, \n                                   loss='huber', random_state =42)\nscore = rmsle_cv(GBoost)\nprint(\"Gradient Boosting score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:32:24.379248Z","iopub.execute_input":"2022-07-10T12:32:24.380298Z","iopub.status.idle":"2022-07-10T12:33:41.427018Z","shell.execute_reply.started":"2022-07-10T12:32:24.380237Z","shell.execute_reply":"2022-07-10T12:33:41.425829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_xgb = xgb.XGBRegressor(colsample_bytree=0.44, gamma=0.045, \n                             learning_rate=0.05, max_depth=3, \n                             min_child_weight=1.7817, n_estimators=2000,\n                             reg_alpha=0.46, reg_lambda=0.858,\n                             subsample=0.5, random_state =42,\n                             nthread = -1)\nscore = rmsle_cv(model_xgb)\nprint(\"Xgboost score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:33:41.428311Z","iopub.execute_input":"2022-07-10T12:33:41.428586Z","iopub.status.idle":"2022-07-10T12:34:34.068846Z","shell.execute_reply.started":"2022-07-10T12:33:41.428561Z","shell.execute_reply":"2022-07-10T12:34:34.067942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,\n                              learning_rate=0.05, n_estimators=720,\n                              max_bin = 55, bagging_fraction = 0.8,\n                              bagging_freq = 5, feature_fraction = 0.2,\n                              feature_fraction_seed=9, bagging_seed=9,\n                              min_data_in_leaf =3, min_sum_hessian_in_leaf = 9,\n                              verbose=-1)\nscore = rmsle_cv(model_lgb)\nprint(\"LGBM score: {:.4f} ({:.4f})\\n\" .format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:34:34.072865Z","iopub.execute_input":"2022-07-10T12:34:34.074943Z","iopub.status.idle":"2022-07-10T12:34:36.316959Z","shell.execute_reply.started":"2022-07-10T12:34:34.074905Z","shell.execute_reply":"2022-07-10T12:34:36.315884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, models):\n        self.models = models\n        \n    # we define clones of the original models to fit the data in\n    def fit(self, X, y):\n        self.models_ = [clone(x) for x in self.models]\n        \n        # Train cloned base models\n        for model in self.models_:\n            model.fit(X, y)\n\n        return self\n    \n    #Now we do the predictions for cloned models and average them\n    def predict(self, X):\n        predictions = np.column_stack([\n            model.predict(X) for model in self.models_\n        ])\n        return np.mean(predictions, axis=1)   ","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:34:36.321448Z","iopub.execute_input":"2022-07-10T12:34:36.321818Z","iopub.status.idle":"2022-07-10T12:34:36.332656Z","shell.execute_reply.started":"2022-07-10T12:34:36.321769Z","shell.execute_reply":"2022-07-10T12:34:36.331676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_models = AveragingModels(models = (ENet, GBoost, KRR, lasso))\n\nscore = rmsle_cv(averaged_models)\nprint(\" Averaged base models score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:34:36.334093Z","iopub.execute_input":"2022-07-10T12:34:36.334505Z","iopub.status.idle":"2022-07-10T12:35:55.298977Z","shell.execute_reply.started":"2022-07-10T12:34:36.334464Z","shell.execute_reply":"2022-07-10T12:35:55.297665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, base_models, meta_model, n_folds=5):\n        self.base_models = base_models\n        self.meta_model = meta_model\n        self.n_folds = n_folds\n   \n    # We again fit the data on clones of the original models\n    def fit(self, X, y):\n        self.base_models_ = [list() for x in self.base_models]\n        self.meta_model_ = clone(self.meta_model)\n        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)\n        \n        # Train cloned base models then create out-of-fold predictions\n        # that are needed to train the cloned meta-model\n        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))\n        for i, model in enumerate(self.base_models):\n            for train_index, holdout_index in kfold.split(X, y):\n                instance = clone(model)\n                self.base_models_[i].append(instance)\n                instance.fit(X[train_index], y[train_index])\n                y_pred = instance.predict(X[holdout_index])\n                out_of_fold_predictions[holdout_index, i] = y_pred\n                \n        # Now train the cloned  meta-model using the out-of-fold predictions as new feature\n        self.meta_model_.fit(out_of_fold_predictions, y)\n        return self\n   \n    #Do the predictions of all base models on the test data and use the averaged predictions as \n    #meta-features for the final prediction which is done by the meta-model\n    def predict(self, X):\n        meta_features = np.column_stack([\n            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)\n            for base_models in self.base_models_ ])\n        return self.meta_model_.predict(meta_features)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:35:55.303012Z","iopub.execute_input":"2022-07-10T12:35:55.307006Z","iopub.status.idle":"2022-07-10T12:35:55.341014Z","shell.execute_reply.started":"2022-07-10T12:35:55.306923Z","shell.execute_reply":"2022-07-10T12:35:55.339555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_averaged_models = StackingAveragedModels(base_models = (ENet, GBoost, KRR),\n                                                 meta_model = lasso)\n\nscore = rmsle_cv(stacked_averaged_models)\nprint(\"Stacking Averaged models score: {:.4f} ({:.4f})\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:35:55.343252Z","iopub.execute_input":"2022-07-10T12:35:55.345376Z","iopub.status.idle":"2022-07-10T12:41:53.88061Z","shell.execute_reply.started":"2022-07-10T12:35:55.345332Z","shell.execute_reply":"2022-07-10T12:41:53.877061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_averaged_models.fit(train_df.values, y_train)\nstacked_train_pred = stacked_averaged_models.predict(train_df.values)\nstacked_pred = np.expm1(stacked_averaged_models.predict(valid_df.values))\nprint(rmsle(y_train, stacked_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:41:53.882225Z","iopub.execute_input":"2022-07-10T12:41:53.882624Z","iopub.status.idle":"2022-07-10T12:43:13.410431Z","shell.execute_reply.started":"2022-07-10T12:41:53.882582Z","shell.execute_reply":"2022-07-10T12:43:13.40695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_models.fit(train_df.values, y_train)\naveraged_train_pred = averaged_models.predict(train_df.values)\naveraged_pred = np.expm1(averaged_models.predict(valid_df.values))\nprint(rmsle(y_train, averaged_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:43:13.41193Z","iopub.execute_input":"2022-07-10T12:43:13.412356Z","iopub.status.idle":"2022-07-10T12:43:30.737774Z","shell.execute_reply.started":"2022-07-10T12:43:13.412315Z","shell.execute_reply":"2022-07-10T12:43:30.733618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_xgb.fit(train_df, y_train)\nxgb_train_pred = model_xgb.predict(train_df)\nxgb_pred = np.expm1(model_xgb.predict(valid_df))\nprint(rmsle(y_train, xgb_train_pred))\n\nmodel_lgb.fit(train_df, y_train)\nlgb_train_pred = model_lgb.predict(train_df)\nlgb_pred = np.expm1(model_lgb.predict(valid_df.values))\nprint(rmsle(y_train, lgb_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:43:30.739596Z","iopub.execute_input":"2022-07-10T12:43:30.740006Z","iopub.status.idle":"2022-07-10T12:43:41.149644Z","shell.execute_reply.started":"2022-07-10T12:43:30.739964Z","shell.execute_reply":"2022-07-10T12:43:41.148506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble = stacked_pred*0.70 + xgb_pred*0.1 + lgb_pred*0.2","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:43:41.151166Z","iopub.execute_input":"2022-07-10T12:43:41.152332Z","iopub.status.idle":"2022-07-10T12:43:41.157754Z","shell.execute_reply.started":"2022-07-10T12:43:41.152289Z","shell.execute_reply":"2022-07-10T12:43:41.156734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Top 10%","metadata":{}},{"cell_type":"code","source":"# sub = pd.DataFrame()\n# sub['Id'] = valid_ID\n# sub['SalePrice'] = lgb_pred\n# sub.to_csv('sub.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-10T12:43:41.159236Z","iopub.execute_input":"2022-07-10T12:43:41.159826Z","iopub.status.idle":"2022-07-10T12:43:41.167924Z","shell.execute_reply.started":"2022-07-10T12:43:41.159768Z","shell.execute_reply":"2022-07-10T12:43:41.166913Z"},"trusted":true},"execution_count":null,"outputs":[]}]}