{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Getting Started","metadata":{}},{"cell_type":"code","source":"!pip install pycaret","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:30.745050Z","iopub.execute_input":"2022-07-21T02:24:30.745577Z","iopub.status.idle":"2022-07-21T02:24:39.279664Z","shell.execute_reply.started":"2022-07-21T02:24:30.745534Z","shell.execute_reply":"2022-07-21T02:24:39.278245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('max_columns', None)\npd.set_option('max_rows', 90)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style('darkgrid')\n\nfrom sklearn.neighbors import KNeighborsRegressor\nimport scipy.stats\nfrom sklearn.preprocessing import StandardScaler\nfrom pycaret.regression import setup, compare_models\nfrom sklearn.model_selection import KFold, cross_val_score\n\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import BayesianRidge, HuberRegressor, Ridge, OrthogonalMatchingPursuit\nfrom lightgbm import LGBMRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom xgboost import XGBRegressor\n\nimport optuna","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:39.281996Z","iopub.execute_input":"2022-07-21T02:24:39.282462Z","iopub.status.idle":"2022-07-21T02:24:45.242295Z","shell.execute_reply.started":"2022-07-21T02:24:39.282413Z","shell.execute_reply":"2022-07-21T02:24:45.241060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train0 = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ntest0 = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\nsample_submission = pd.read_csv('../input/house-prices-advanced-regression-techniques/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.244421Z","iopub.execute_input":"2022-07-21T02:24:45.244722Z","iopub.status.idle":"2022-07-21T02:24:45.334143Z","shell.execute_reply.started":"2022-07-21T02:24:45.244692Z","shell.execute_reply":"2022-07-21T02:24:45.333232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train0","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.335965Z","iopub.execute_input":"2022-07-21T02:24:45.336403Z","iopub.status.idle":"2022-07-21T02:24:45.449326Z","shell.execute_reply.started":"2022-07-21T02:24:45.336360Z","shell.execute_reply":"2022-07-21T02:24:45.448289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test0","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.450721Z","iopub.execute_input":"2022-07-21T02:24:45.451023Z","iopub.status.idle":"2022-07-21T02:24:45.565588Z","shell.execute_reply.started":"2022-07-21T02:24:45.450994Z","shell.execute_reply":"2022-07-21T02:24:45.564530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.566881Z","iopub.execute_input":"2022-07-21T02:24:45.567261Z","iopub.status.idle":"2022-07-21T02:24:45.580333Z","shell.execute_reply.started":"2022-07-21T02:24:45.567175Z","shell.execute_reply":"2022-07-21T02:24:45.579196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine Train and Test Sets","metadata":{}},{"cell_type":"code","source":"target = train0['SalePrice']\ntest_ids = test0['Id']\n\ntrain1 = train0.drop(['Id', 'SalePrice'], axis=1)\ntest1 = test0.drop('Id', axis=1)\n\ndata1 = pd.concat([train1, test1], axis=0).reset_index(drop=True)\ndata1","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.581575Z","iopub.execute_input":"2022-07-21T02:24:45.581881Z","iopub.status.idle":"2022-07-21T02:24:45.721392Z","shell.execute_reply.started":"2022-07-21T02:24:45.581851Z","shell.execute_reply":"2022-07-21T02:24:45.720324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.724081Z","iopub.execute_input":"2022-07-21T02:24:45.724624Z","iopub.status.idle":"2022-07-21T02:24:45.733881Z","shell.execute_reply.started":"2022-07-21T02:24:45.724578Z","shell.execute_reply":"2022-07-21T02:24:45.732726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning","metadata":{}},{"cell_type":"code","source":"data2 = data1.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.735684Z","iopub.execute_input":"2022-07-21T02:24:45.735985Z","iopub.status.idle":"2022-07-21T02:24:45.744759Z","shell.execute_reply.started":"2022-07-21T02:24:45.735957Z","shell.execute_reply":"2022-07-21T02:24:45.743757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensure Proper Data Types","metadata":{}},{"cell_type":"code","source":"data2['MSSubClass'] = data2['MSSubClass'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.745934Z","iopub.execute_input":"2022-07-21T02:24:45.746237Z","iopub.status.idle":"2022-07-21T02:24:45.758797Z","shell.execute_reply.started":"2022-07-21T02:24:45.746209Z","shell.execute_reply":"2022-07-21T02:24:45.757725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fill Categorical Missing Values","metadata":{}},{"cell_type":"code","source":"# Impute using a constant value\nfor column in [\n    'Alley',\n    'BsmtQual',\n    'BsmtCond',\n    'BsmtExposure',\n    'BsmtFinType1',\n    'BsmtFinType2',\n    'FireplaceQu',\n    'GarageType',\n    'GarageFinish',\n    'GarageQual',\n    'GarageCond',\n    'PoolQC',\n    'Fence',\n    'MiscFeature'\n]:\n    data2[column] = data2[column].fillna(\"None\")\n\n# Impute using the column mode\nfor column in [\n    'MSZoning',\n    'Utilities',\n    'Exterior1st',\n    'Exterior2nd',\n    'MasVnrType',\n    'Electrical',\n    'KitchenQual',\n    'Functional',\n    'SaleType'\n]:\n    data2[column] = data2[column].fillna(data2[column].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:24:45.760853Z","iopub.execute_input":"2022-07-21T02:24:45.761468Z","iopub.status.idle":"2022-07-21T02:24:45.796137Z","shell.execute_reply.started":"2022-07-21T02:24:45.761419Z","shell.execute_reply":"2022-07-21T02:24:45.794858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:25:57.725783Z","iopub.execute_input":"2022-07-21T02:25:57.726351Z","iopub.status.idle":"2022-07-21T02:25:57.770452Z","shell.execute_reply.started":"2022-07-21T02:25:57.726302Z","shell.execute_reply":"2022-07-21T02:25:57.769369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2.BsmtQual.replace(['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5], inplace=True)\ndata2.BsmtCond.replace(['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5], inplace=True)\ndata2.ExterQual.replace(['Po','Fa','TA','Gd','Ex'],[1,2,3,4,5], inplace=True)\ndata2.ExterCond.replace(['Po','Fa','TA','Gd','Ex'],[1,2,3,4,5], inplace=True)\ndata2.GarageQual.replace(['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5], inplace=True)\ndata2.GarageCond.replace(['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5], inplace=True)\ndata2.HeatingQC.replace(['Po','Fa','TA','Gd','Ex'],[1,2,3,4,5], inplace=True)\ndata2.CentralAir.replace(['N','Y'],[0,1], inplace=True)\ndata2.KitchenQual.replace(['Po','Fa','TA','Gd','Ex'],[1,2,3,4,5], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:02.701342Z","iopub.execute_input":"2022-07-21T02:27:02.701759Z","iopub.status.idle":"2022-07-21T02:27:02.748145Z","shell.execute_reply.started":"2022-07-21T02:27:02.701720Z","shell.execute_reply":"2022-07-21T02:27:02.747324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data3 = data2.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:07.077330Z","iopub.execute_input":"2022-07-21T02:27:07.077972Z","iopub.status.idle":"2022-07-21T02:27:07.084219Z","shell.execute_reply.started":"2022-07-21T02:27:07.077933Z","shell.execute_reply":"2022-07-21T02:27:07.083374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Numeric Missing Values","metadata":{}},{"cell_type":"code","source":"def knn_impute(df, na_target):\n    df = df.copy()\n    \n    numeric_df = df.select_dtypes(np.number)\n    non_na_columns = numeric_df.loc[: ,numeric_df.isna().sum() == 0].columns\n    \n    y_train = numeric_df.loc[numeric_df[na_target].isna() == False, na_target]\n    X_train = numeric_df.loc[numeric_df[na_target].isna() == False, non_na_columns]\n    X_test = numeric_df.loc[numeric_df[na_target].isna() == True, non_na_columns]\n    \n    knn = KNeighborsRegressor()\n    knn.fit(X_train, y_train)\n    \n    y_pred = knn.predict(X_test)\n    \n    df.loc[df[na_target].isna() == True, na_target] = y_pred\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:07.362073Z","iopub.execute_input":"2022-07-21T02:27:07.362443Z","iopub.status.idle":"2022-07-21T02:27:07.370780Z","shell.execute_reply.started":"2022-07-21T02:27:07.362413Z","shell.execute_reply":"2022-07-21T02:27:07.369408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in [\n    'LotFrontage',\n    'MasVnrArea',\n    'BsmtFinSF1',\n    'BsmtFinSF2',\n    'BsmtUnfSF',\n    'TotalBsmtSF',\n    'BsmtFullBath',\n    'BsmtHalfBath',\n    'GarageYrBlt',\n    'GarageCars',\n    'GarageArea'\n]:\n    data3 = knn_impute(data3, column)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:07.523396Z","iopub.execute_input":"2022-07-21T02:27:07.524054Z","iopub.status.idle":"2022-07-21T02:27:07.839374Z","shell.execute_reply.started":"2022-07-21T02:27:07.524018Z","shell.execute_reply":"2022-07-21T02:27:07.838282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data4 = data3.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:07.841094Z","iopub.execute_input":"2022-07-21T02:27:07.841599Z","iopub.status.idle":"2022-07-21T02:27:07.847507Z","shell.execute_reply.started":"2022-07-21T02:27:07.841553Z","shell.execute_reply":"2022-07-21T02:27:07.846527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"data4[\"SqFtPerRoom\"] = data4[\"GrLivArea\"] / (data4[\"TotRmsAbvGrd\"] +\n                                                       data4[\"FullBath\"] +\n                                                       data4[\"HalfBath\"] +\n                                                       data4[\"KitchenAbvGr\"])\n\ndata4['Total_Home_Quality'] = data4['OverallQual'] + data4['OverallCond']\n\ndata4['Total_Bathrooms'] = (data4['FullBath'] + (0.5 * data4['HalfBath']) +\n                               data4['BsmtFullBath'] + (0.5 * data4['BsmtHalfBath']))\n\ndata4[\"HighQualSF\"] = data4[\"1stFlrSF\"] + data4[\"2ndFlrSF\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:07.908632Z","iopub.execute_input":"2022-07-21T02:27:07.909207Z","iopub.status.idle":"2022-07-21T02:27:07.921896Z","shell.execute_reply.started":"2022-07-21T02:27:07.909140Z","shell.execute_reply":"2022-07-21T02:27:07.920591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data5 = data4.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:08.023493Z","iopub.execute_input":"2022-07-21T02:27:08.024056Z","iopub.status.idle":"2022-07-21T02:27:08.030375Z","shell.execute_reply.started":"2022-07-21T02:27:08.024008Z","shell.execute_reply":"2022-07-21T02:27:08.029619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Transformations","metadata":{}},{"cell_type":"markdown","source":"## Log Transform for Skewed Features","metadata":{}},{"cell_type":"code","source":"skew_df = pd.DataFrame(data5.select_dtypes(np.number).columns, columns=['Feature'])\nskew_df['Skew'] = skew_df['Feature'].apply(lambda feature: scipy.stats.skew(data5[feature]))\nskew_df['Absolute Skew'] = skew_df['Skew'].apply(abs)\nskew_df['Skewed'] = skew_df['Absolute Skew'].apply(lambda x: True if x >= 0.5 else False)\nskew_df","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:08.436353Z","iopub.execute_input":"2022-07-21T02:27:08.436914Z","iopub.status.idle":"2022-07-21T02:27:08.487462Z","shell.execute_reply.started":"2022-07-21T02:27:08.436864Z","shell.execute_reply":"2022-07-21T02:27:08.486766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in skew_df.query(\"Skewed == True\")['Feature'].values:\n    data5[column] = np.log1p(data5[column])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:08.583963Z","iopub.execute_input":"2022-07-21T02:27:08.584556Z","iopub.status.idle":"2022-07-21T02:27:08.616383Z","shell.execute_reply.started":"2022-07-21T02:27:08.584502Z","shell.execute_reply":"2022-07-21T02:27:08.615248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cosine Transform for Cyclical Features","metadata":{}},{"cell_type":"code","source":"data4['MoSold'] = (-np.cos(0.5236 * data5['MoSold']))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:08.975785Z","iopub.execute_input":"2022-07-21T02:27:08.976170Z","iopub.status.idle":"2022-07-21T02:27:08.982940Z","shell.execute_reply.started":"2022-07-21T02:27:08.976121Z","shell.execute_reply":"2022-07-21T02:27:08.982240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data6 = data5.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:09.034833Z","iopub.execute_input":"2022-07-21T02:27:09.035425Z","iopub.status.idle":"2022-07-21T02:27:09.040069Z","shell.execute_reply.started":"2022-07-21T02:27:09.035376Z","shell.execute_reply":"2022-07-21T02:27:09.039307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encode Categoricals","metadata":{}},{"cell_type":"code","source":"data6 = pd.get_dummies(data6)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:09.369718Z","iopub.execute_input":"2022-07-21T02:27:09.370318Z","iopub.status.idle":"2022-07-21T02:27:09.425380Z","shell.execute_reply.started":"2022-07-21T02:27:09.370272Z","shell.execute_reply":"2022-07-21T02:27:09.424318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data7 = data6.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:09.505248Z","iopub.execute_input":"2022-07-21T02:27:09.505621Z","iopub.status.idle":"2022-07-21T02:27:09.510762Z","shell.execute_reply.started":"2022-07-21T02:27:09.505588Z","shell.execute_reply":"2022-07-21T02:27:09.509710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scaling","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(data7)\n\ndata7 = pd.DataFrame(scaler.transform(data7), index=data7.index, columns=data7.columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:09.760657Z","iopub.execute_input":"2022-07-21T02:27:09.761023Z","iopub.status.idle":"2022-07-21T02:27:09.811326Z","shell.execute_reply.started":"2022-07-21T02:27:09.760993Z","shell.execute_reply":"2022-07-21T02:27:09.810134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data7","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:09.925870Z","iopub.execute_input":"2022-07-21T02:27:09.926265Z","iopub.status.idle":"2022-07-21T02:27:10.238887Z","shell.execute_reply.started":"2022-07-21T02:27:09.926234Z","shell.execute_reply":"2022-07-21T02:27:10.238146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data8 = data7.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:10.240223Z","iopub.execute_input":"2022-07-21T02:27:10.240652Z","iopub.status.idle":"2022-07-21T02:27:10.246017Z","shell.execute_reply.started":"2022-07-21T02:27:10.240617Z","shell.execute_reply":"2022-07-21T02:27:10.245285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target Transformation","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 10))\n\nplt.subplot(1, 2, 1)\nsns.distplot(target, kde=True, fit=scipy.stats.norm)\nplt.title(\"Without Log Transform\")\n\nplt.subplot(1, 2, 2)\nsns.distplot(np.log(target), kde=True, fit=scipy.stats.norm)\nplt.xlabel(\"Log SalePrice\")\nplt.title(\"With Log Transform\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:10.411976Z","iopub.execute_input":"2022-07-21T02:27:10.412514Z","iopub.status.idle":"2022-07-21T02:27:11.332293Z","shell.execute_reply.started":"2022-07-21T02:27:10.412479Z","shell.execute_reply":"2022-07-21T02:27:11.331075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_target = np.log(target)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:11.333782Z","iopub.execute_input":"2022-07-21T02:27:11.334284Z","iopub.status.idle":"2022-07-21T02:27:11.338556Z","shell.execute_reply.started":"2022-07-21T02:27:11.334239Z","shell.execute_reply":"2022-07-21T02:27:11.337783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split Data","metadata":{}},{"cell_type":"code","source":"train_final = data8.loc[:train0.index.max(), :].copy()\ntest_final = data8.loc[train0.index.max() + 1:, :].reset_index(drop=True).copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:11.340508Z","iopub.execute_input":"2022-07-21T02:27:11.341109Z","iopub.status.idle":"2022-07-21T02:27:11.358791Z","shell.execute_reply.started":"2022-07-21T02:27:11.341062Z","shell.execute_reply":"2022-07-21T02:27:11.357960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:11.360693Z","iopub.execute_input":"2022-07-21T02:27:11.361151Z","iopub.status.idle":"2022-07-21T02:27:11.675136Z","shell.execute_reply.started":"2022-07-21T02:27:11.361105Z","shell.execute_reply":"2022-07-21T02:27:11.673976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:11.676676Z","iopub.execute_input":"2022-07-21T02:27:11.677305Z","iopub.status.idle":"2022-07-21T02:27:11.995961Z","shell.execute_reply.started":"2022-07-21T02:27:11.677258Z","shell.execute_reply":"2022-07-21T02:27:11.995040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Selection","metadata":{}},{"cell_type":"code","source":"# _ = setup(data=pd.concat([train_final, log_target], axis=1), target='SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:11.997586Z","iopub.execute_input":"2022-07-21T02:27:11.997878Z","iopub.status.idle":"2022-07-21T02:27:12.001569Z","shell.execute_reply.started":"2022-07-21T02:27:11.997850Z","shell.execute_reply":"2022-07-21T02:27:12.000477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare_models()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.003046Z","iopub.execute_input":"2022-07-21T02:27:12.003369Z","iopub.status.idle":"2022-07-21T02:27:12.014179Z","shell.execute_reply.started":"2022-07-21T02:27:12.003340Z","shell.execute_reply":"2022-07-21T02:27:12.012926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter Optimization","metadata":{}},{"cell_type":"code","source":"# def br_objective(trial):\n#     n_iter = trial.suggest_int('n_iter', 50, 600)\n#     tol = trial.suggest_loguniform('tol', 1e-8, 10.0)\n#     alpha_1 = trial.suggest_loguniform('alpha_1', 1e-8, 10.0)\n#     alpha_2 = trial.suggest_loguniform('alpha_2', 1e-8, 10.0)\n#     lambda_1 = trial.suggest_loguniform('lambda_1', 1e-8, 10.0)\n#     lambda_2 = trial.suggest_loguniform('lambda_2', 1e-8, 10.0)\n    \n#     model = BayesianRidge(\n#         n_iter=n_iter,\n#         tol=tol,\n#         alpha_1=alpha_1,\n#         alpha_2=alpha_2,\n#         lambda_1=lambda_1,\n#         lambda_2=lambda_2\n#     )\n    \n#     model.fit(train_final, log_target)\n    \n#     cv_scores = np.exp(np.sqrt(-cross_val_score(model, train_final, log_target, scoring='neg_mean_squared_error', cv=kf)))\n    \n#     return np.mean(cv_scores)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.282066Z","iopub.execute_input":"2022-07-21T02:27:12.282487Z","iopub.status.idle":"2022-07-21T02:27:12.287061Z","shell.execute_reply.started":"2022-07-21T02:27:12.282457Z","shell.execute_reply":"2022-07-21T02:27:12.286072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study = optuna.create_study(direction='minimize')\n# study.optimize(br_objective, n_trials=100)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.303980Z","iopub.execute_input":"2022-07-21T02:27:12.304707Z","iopub.status.idle":"2022-07-21T02:27:12.310234Z","shell.execute_reply.started":"2022-07-21T02:27:12.304667Z","shell.execute_reply":"2022-07-21T02:27:12.309470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study.best_params","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.413211Z","iopub.execute_input":"2022-07-21T02:27:12.413835Z","iopub.status.idle":"2022-07-21T02:27:12.420082Z","shell.execute_reply.started":"2022-07-21T02:27:12.413798Z","shell.execute_reply":"2022-07-21T02:27:12.419324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bagging Ensemble","metadata":{}},{"cell_type":"code","source":"catboost_params = {\n    'iterations': 6000,\n    'learning_rate': 0.005,\n    'depth': 4,\n    'l2_leaf_reg': 1,\n    'eval_metric':'RMSE',\n    'early_stopping_rounds': 200,\n    'random_seed': 42\n}\n\nbr_params = {\n    'n_iter': 304,\n    'tol': 0.16864712769300896,\n    'alpha_1': 5.589616542154059e-07,\n    'alpha_2': 9.799343618469923,\n    'lambda_1': 1.7735725582463822,\n    'lambda_2': 3.616928181181732e-06\n}\n\nlightgbm_params = {\n    'num_leaves': 39,\n    'max_depth': 2,\n    'learning_rate': 0.13705339989856127,\n    'n_estimators': 273\n}\n\nridge_params = {\n    'alpha': 631.1412445239156\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.938950Z","iopub.execute_input":"2022-07-21T02:27:12.939469Z","iopub.status.idle":"2022-07-21T02:27:12.945427Z","shell.execute_reply.started":"2022-07-21T02:27:12.939438Z","shell.execute_reply":"2022-07-21T02:27:12.944624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {\n    \"catboost\": CatBoostRegressor(**catboost_params, verbose=0),\n    \"br\": BayesianRidge(**br_params),\n    \"lightgbm\": LGBMRegressor(**lightgbm_params),\n    \"ridge\": Ridge(**ridge_params),\n    \"omp\": OrthogonalMatchingPursuit()\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:12.965520Z","iopub.execute_input":"2022-07-21T02:27:12.966080Z","iopub.status.idle":"2022-07-21T02:27:12.976675Z","shell.execute_reply.started":"2022-07-21T02:27:12.966044Z","shell.execute_reply":"2022-07-21T02:27:12.975623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, model in models.items():\n    model.fit(train_final, log_target)\n    print(name + \" trained.\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:13.096826Z","iopub.execute_input":"2022-07-21T02:27:13.097476Z","iopub.status.idle":"2022-07-21T02:27:27.234941Z","shell.execute_reply.started":"2022-07-21T02:27:13.097439Z","shell.execute_reply":"2022-07-21T02:27:27.233253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"results = {}\n\nkf = KFold(n_splits=10)\n\nfor name, model in models.items():\n    result = np.exp(np.sqrt(-cross_val_score(model, train_final, log_target, scoring='neg_mean_squared_error', cv=kf)))\n    results[name] = result","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:27:27.242115Z","iopub.execute_input":"2022-07-21T02:27:27.247606Z","iopub.status.idle":"2022-07-21T02:29:40.478305Z","shell.execute_reply.started":"2022-07-21T02:27:27.247519Z","shell.execute_reply":"2022-07-21T02:29:40.477029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.484859Z","iopub.execute_input":"2022-07-21T02:29:40.488228Z","iopub.status.idle":"2022-07-21T02:29:40.503685Z","shell.execute_reply.started":"2022-07-21T02:29:40.488121Z","shell.execute_reply":"2022-07-21T02:29:40.502300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name, result in results.items():\n    print(\"----------\\n\" + name)\n    print(np.mean(result))\n    print(np.std(result))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.510215Z","iopub.execute_input":"2022-07-21T02:29:40.513276Z","iopub.status.idle":"2022-07-21T02:29:40.530799Z","shell.execute_reply.started":"2022-07-21T02:29:40.510910Z","shell.execute_reply":"2022-07-21T02:29:40.529327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine Predictions","metadata":{}},{"cell_type":"code","source":"final_predictions = (\n    0.4 * np.exp(models['catboost'].predict(test_final)) +\n    0.2 * np.exp(models['br'].predict(test_final)) +\n    0.2 * np.exp(models['lightgbm'].predict(test_final)) +\n    0.1 * np.exp(models['ridge'].predict(test_final)) +\n    0.1 * np.exp(models['omp'].predict(test_final))\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.532917Z","iopub.execute_input":"2022-07-21T02:29:40.533834Z","iopub.status.idle":"2022-07-21T02:29:40.688699Z","shell.execute_reply.started":"2022-07-21T02:29:40.533780Z","shell.execute_reply":"2022-07-21T02:29:40.687468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.690207Z","iopub.execute_input":"2022-07-21T02:29:40.690658Z","iopub.status.idle":"2022-07-21T02:29:40.700817Z","shell.execute_reply.started":"2022-07-21T02:29:40.690613Z","shell.execute_reply":"2022-07-21T02:29:40.699630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.concat([test_ids, pd.Series(final_predictions, name='SalePrice')], axis=1)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.702737Z","iopub.execute_input":"2022-07-21T02:29:40.703683Z","iopub.status.idle":"2022-07-21T02:29:40.728858Z","shell.execute_reply.started":"2022-07-21T02:29:40.703620Z","shell.execute_reply":"2022-07-21T02:29:40.727692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('./submission.csv', index=False, header=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T02:29:40.731536Z","iopub.execute_input":"2022-07-21T02:29:40.732240Z","iopub.status.idle":"2022-07-21T02:29:40.753470Z","shell.execute_reply.started":"2022-07-21T02:29:40.732187Z","shell.execute_reply":"2022-07-21T02:29:40.752188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Livestream Included!\n\n***\n\nThis notebook was created during a YouTube live session.  \nFor an in-depth guide, check it out here!  \nhttps://youtu.be/zwYHloLXH0c","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}