{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Importing Libraries and Datasets","metadata":{}},{"cell_type":"markdown","source":"### 1.1 Importing Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\n\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom category_encoders import MEstimateEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import cross_validate, GridSearchCV\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import Lasso, Ridge\nfrom sklearn.linear_model import ElasticNet, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-14T23:12:05.285905Z","iopub.execute_input":"2022-07-14T23:12:05.286691Z","iopub.status.idle":"2022-07-14T23:12:08.329027Z","shell.execute_reply.started":"2022-07-14T23:12:05.286593Z","shell.execute_reply":"2022-07-14T23:12:08.326883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Loading Datasets","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/train.csv\")\ndf_test = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/test.csv\")\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:21:11.118521Z","iopub.execute_input":"2022-07-14T23:21:11.118915Z","iopub.status.idle":"2022-07-14T23:21:11.230189Z","shell.execute_reply.started":"2022-07-14T23:21:11.118885Z","shell.execute_reply":"2022-07-14T23:21:11.228780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see some missing values in this dataset. Let's explore the dataset to handle this.","metadata":{}},{"cell_type":"markdown","source":"## 2. Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"Feature engineering is the process of selecting, manipulating, and transforming raw data into features that can be used to create a predictive model using machine learning or statistical modeling. ","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Missing Value Handling","metadata":{}},{"cell_type":"markdown","source":"#### 2.1.1 Checking Missing Values","metadata":{}},{"cell_type":"code","source":"train_null = df_train.isna().sum()\ntest_null = df_test.isna().sum()\nmissing = pd.DataFrame(\n              data=[train_null, train_null/df_train.shape[0]*100,\n                    test_null, test_null/df_test.shape[0]*100],\n              columns=df_train.columns,\n              index=[\"Train Null\", \"Train Null (%)\", \"Test Null\", \"Test Null (%)\"]\n          ).T.sort_values([\"Train Null\", \"Test Null\"], ascending=False)\n\n# Filter only columns with missing values\nmissing = missing.loc[(missing[\"Train Null\"] > 0) | (missing[\"Test Null\"] > 0)]\nmissing.style.background_gradient('summer_r')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:28.105774Z","iopub.execute_input":"2022-07-14T23:28:28.106201Z","iopub.status.idle":"2022-07-14T23:28:28.229004Z","shell.execute_reply.started":"2022-07-14T23:28:28.106169Z","shell.execute_reply":"2022-07-14T23:28:28.227926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Wow, there are a lot of missing values in this dataset. The missing values are not only found in the training data, but also in the test data. ","metadata":{}},{"cell_type":"code","source":"# Plot variables with more than 5% missing values\nmissing.loc[missing[\"Train Null (%)\"] > 5, [\"Train Null (%)\", \"Test Null (%)\"]].iloc[::-1].plot.barh(figsize=(8,6))\nplt.title(\"Variables With More Than 5% Missing Values\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:28.839372Z","iopub.execute_input":"2022-07-14T23:28:28.839791Z","iopub.status.idle":"2022-07-14T23:28:29.200966Z","shell.execute_reply.started":"2022-07-14T23:28:28.839755Z","shell.execute_reply":"2022-07-14T23:28:29.199951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we have 11 variables with more than 5% of missing values in it. These variables are PoolQC, MiscFeature, Alley, Fence, FireplaceQu, LotFrontage, GarageYrBlt, GarageFinish, GarageQual, GarageCond, and GarageType. The ratio of missing values for each variable in the train data and test data looks quite balanced. We will handle this problem. First, let's group each variable with missing values based on their data type.","metadata":{}},{"cell_type":"markdown","source":"#### 2.1.2 Grouping Variable with Missing Values","metadata":{}},{"cell_type":"code","source":"df_missing = df_train[missing.index]\nmissing_cat = df_missing.loc[:, df_missing.dtypes == \"object\"].columns\nmissing_num = df_missing.loc[:, df_missing.dtypes != \"object\"].columns\n\nprint(f\"number of categorical variables with missing values: {len(missing_cat)}\")\nprint(f\"number of numerical variables with missing values: {len(missing_num)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:30.615047Z","iopub.execute_input":"2022-07-14T23:28:30.617288Z","iopub.status.idle":"2022-07-14T23:28:30.628068Z","shell.execute_reply.started":"2022-07-14T23:28:30.617252Z","shell.execute_reply":"2022-07-14T23:28:30.626895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we have 23 categorical variables and 11 numerical variables with missing values. Now let's check their distribution.","metadata":{}},{"cell_type":"markdown","source":"#### 2.1.3 Checking The Distribution of Variables with Missing Values","metadata":{}},{"cell_type":"markdown","source":"#### Categorical Variables","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(6, 4, figsize=(20, 24))\nax = ax.flatten()\n\nfor i, var in enumerate(missing_cat):\n    count = sns.countplot(data=df_train, x=var, ax=ax[i])\n    for bar in count.patches:\n        count.annotate(format(bar.get_height()),\n            (bar.get_x() + bar.get_width() / 2,\n            bar.get_height()), ha='center', va='center',\n            size=11, xytext=(0, 8),\n            textcoords='offset points')\n        \n    ax[i].set_title(f\"{var} Distribution\")\n    if df_train[var].nunique() > 6:\n        ax[i].tick_params(axis='x', rotation=45)\n    \nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:34.832957Z","iopub.execute_input":"2022-07-14T23:28:34.833345Z","iopub.status.idle":"2022-07-14T23:28:39.110906Z","shell.execute_reply.started":"2022-07-14T23:28:34.833315Z","shell.execute_reply":"2022-07-14T23:28:39.109759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Numerical Variables","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 4, figsize=(20, 10))\nax = ax.flatten()\n\nfor i, var in enumerate(missing_num):\n    sns.histplot(data=df_train, x=var, kde=True, ax=ax[i])\n    ax[i].set_title(f\"{var} Distribution\")\n\nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:39.112559Z","iopub.execute_input":"2022-07-14T23:28:39.112891Z","iopub.status.idle":"2022-07-14T23:28:42.133688Z","shell.execute_reply.started":"2022-07-14T23:28:39.112839Z","shell.execute_reply":"2022-07-14T23:28:42.132613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.1.4 Filling Missing Values","metadata":{}},{"cell_type":"markdown","source":"The most important thing to do to be able to handle missing values in this dataset is understanding the description of each variable first. To do this, we can read **data_description.txt** file provided in this competition. ","metadata":{}},{"cell_type":"markdown","source":"#### Categorical","metadata":{}},{"cell_type":"markdown","source":"After reading data description file, i decided to fill in the missing values as follows:\n1. Fill with **None**: PoolQC, MiscFeature, Alley, Fence, FireplaceQu, GarageFinish, GarageQual, GarageCond, GarageType\n2. Fill with **NB** (No Basement): BsmtExposure, BsmtFinType2, BsmtCond, BsmtQual, BsmtFinType1\n3. Fill with **Mode**: Electrical, Functional, KitchenQual, Exterior1st, Exterior2nd, MSZoning, SaleType, MasVnrType\n\nFor Utilities variable, we will just remove it because this variable does not provide information at all. As we can see in the bar chart above, this variable is only dominated by 1 value.","metadata":{}},{"cell_type":"code","source":"cat_none_var = [\"PoolQC\", \"MiscFeature\", \"Alley\", \"Fence\", \"FireplaceQu\", \"GarageFinish\", \"GarageQual\", \"GarageCond\", \"GarageType\"]\ncat_nb_var = [\"BsmtExposure\", \"BsmtFinType2\", \"BsmtCond\", \"BsmtQual\", \"BsmtFinType1\"]\ncat_mode_var = [\"Electrical\", \"Functional\", \"KitchenQual\", \"Exterior1st\", \"Exterior2nd\", \"MSZoning\", \"SaleType\", \"MasVnrType\"]\ncat_mode = {var: df_train[var].mode()[0] for var in cat_mode_var}\n\n# Categorical\nfor df in [df_train, df_test]:\n    # Fill with None (because null means no building/object)\n    df[cat_none_var] = df[cat_none_var].fillna(\"None\")\n    \n    # Fill with NB (because null means no basememt)\n    df[cat_nb_var] = df[cat_nb_var].fillna(\"NB\")\n    \n    # Fill other categorical variables with mode\n    for var in cat_mode_var:\n        df[var].fillna(cat_mode[var], inplace=True)\n        \n    # Drop variable because no information\n    df.drop(\"Utilities\", axis=1, inplace=True)\n\nmissing_cat = missing_cat.drop(\"Utilities\")\nprint(f\"Categorical variable missing values in train data: {df_train[missing_cat].isna().sum().sum()}\")\nprint(f\"Categorical variable missing values in test data: {df_test[missing_cat].isna().sum().sum()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:44.783153Z","iopub.execute_input":"2022-07-14T23:28:44.783716Z","iopub.status.idle":"2022-07-14T23:28:44.831168Z","shell.execute_reply.started":"2022-07-14T23:28:44.783682Z","shell.execute_reply":"2022-07-14T23:28:44.830426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Numerical","metadata":{}},{"cell_type":"markdown","source":"For numerical variables, we will just fill the missing values with 0 because most of this variables are related to buildings or objects which may not exist. But for LotFrontage variable, we will fill the missing values with the average value of the variable based on the type of road access to property (Street).","metadata":{}},{"cell_type":"code","source":"mean_LF = df_train.groupby(\"Street\")[\"LotFrontage\"].mean()\nnum_zero_var = missing_num.drop(\"LotFrontage\")\n\nfor df in [df_train, df_test]:\n    df.loc[(df[\"LotFrontage\"].isna()) & (df[\"Street\"] == \"Grvl\"), \"LotFrontage\"] = mean_LF[\"Grvl\"]\n    df.loc[(df[\"LotFrontage\"].isna()) & (df[\"Street\"] == \"Pave\"), \"LotFrontage\"] = mean_LF[\"Pave\"]\n    \n    for var in num_zero_var:\n        df[var].fillna(0, inplace=True)\n        \nprint(f\"Numerical variable missing values in train data: {df_train[missing_num].isna().sum().sum()}\")\nprint(f\"Numerical variable missing values in test data: {df_test[missing_num].isna().sum().sum()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:46.901006Z","iopub.execute_input":"2022-07-14T23:28:46.901686Z","iopub.status.idle":"2022-07-14T23:28:46.927287Z","shell.execute_reply.started":"2022-07-14T23:28:46.901651Z","shell.execute_reply":"2022-07-14T23:28:46.926538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Done. We don't have any missing values now.","metadata":{}},{"cell_type":"markdown","source":"### 2.2 Categorical Variable Encoding","metadata":{}},{"cell_type":"markdown","source":"Categorical variables need to be converted into numerical format so that the data with converted categorical values can be provided to the predictive models. There are two types of categorical variable, which is nominal and ordinal. Nominal variable has no intrinsic ordering to its categories, while ordinal variable has a clear ordering. There are different ways to encode these variables depending on their data type. Let's check our categorical variable values first.","metadata":{}},{"cell_type":"code","source":"# Get numerical and categorical variable\ncat_var = df_train.loc[:, df_train.dtypes == \"object\"].nunique() # Get variable names and number of unique values\nnum_var = df_train.loc[:, df_train.dtypes != \"object\"].columns # Get variable names","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:50.954935Z","iopub.execute_input":"2022-07-14T23:28:50.955581Z","iopub.status.idle":"2022-07-14T23:28:50.974458Z","shell.execute_reply.started":"2022-07-14T23:28:50.955549Z","shell.execute_reply":"2022-07-14T23:28:50.973663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.2.1 Get Unique Values For Each Categorical Variable ","metadata":{}},{"cell_type":"code","source":"# Count sorted unique values (alphabetically) for each categorical variable\ncat_var_unique = {var: sorted(df_train[var].unique()) for var in cat_var.index}\n\n# Add \"-\" for each values to replace none in the DataFrame (25 is highest len of unique values)\nfor key, val in cat_var_unique.items():\n    cat_var_unique[key] += [\"-\" for x in range(25-len(val))]\n\npd.DataFrame.from_dict(cat_var_unique, orient=\"index\").sort_values([x for x in range(25)])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:53.476529Z","iopub.execute_input":"2022-07-14T23:28:53.476920Z","iopub.status.idle":"2022-07-14T23:28:53.547829Z","shell.execute_reply.started":"2022-07-14T23:28:53.476887Z","shell.execute_reply":"2022-07-14T23:28:53.546788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the table above, we can see the unique values of each categorical variable in this dataset. Now we can determine which variables have nominal values and which variables have ordinal values, and then decide what method we will use to encode that variable.","metadata":{}},{"cell_type":"markdown","source":"#### 2.2.2 Ordinal Encoding","metadata":{}},{"cell_type":"markdown","source":"For ordinal variables, we will use ordinal encoding to encode these variables. Ordinal encoding converts each label into integer values and the encoded data represents the sequence of labels. We will need to group each ordinal variable based on their unique values.","metadata":{}},{"cell_type":"code","source":"ord_var1 = [\"ExterCond\", \"HeatingQC\"]\nord_var1_cat = [\"Po\", \"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\nord_var2 = [\"ExterQual\", \"KitchenQual\"]\nord_var2_cat = [\"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\nord_var3 = [\"FireplaceQu\", \"GarageQual\", \"GarageCond\"]\nord_var3_cat = [\"None\", \"Po\", \"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\nord_var4 = [\"BsmtQual\"]\nord_var4_cat = [\"NB\", \"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\nord_var5 = [\"BsmtCond\"]\nord_var5_cat = [\"NB\", \"Po\", \"Fa\", \"TA\", \"Gd\"]\n\nord_var6 = [\"BsmtExposure\"]\nord_var6_cat = [\"NB\", \"No\", \"Mn\", \"Av\", \"Gd\"]\n\nord_var7 = [\"BsmtFinType1\", \"BsmtFinType2\"]\nord_var7_cat = [\"NB\", \"Unf\", \"LwQ\", \"Rec\", \"BLQ\", \"ALQ\", \"GLQ\"]\n\n# Put all in one array for easier iteration\nord_var = [ord_var1, ord_var2, ord_var3, ord_var4, ord_var5, ord_var6, ord_var7]\nord_var_cat = [ord_var1_cat, ord_var2_cat, ord_var3_cat, ord_var4_cat, ord_var5_cat, ord_var6_cat, ord_var7_cat]\nord_all = ord_var1 + ord_var2 + ord_var3 + ord_var4 + ord_var5 + ord_var6 + ord_var7 \n\nfor i in range(len(ord_var)):\n    enc = OrdinalEncoder(categories=[ord_var_cat[i]])\n    for var in ord_var[i]:\n        df_train[var] = enc.fit_transform(df_train[[var]])\n        df_test[var] = enc.transform(df_test[[var]])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:55.806542Z","iopub.execute_input":"2022-07-14T23:28:55.806947Z","iopub.status.idle":"2022-07-14T23:28:55.871424Z","shell.execute_reply.started":"2022-07-14T23:28:55.806913Z","shell.execute_reply":"2022-07-14T23:28:55.870547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is the result.","metadata":{}},{"cell_type":"code","source":"df_train[ord_all]","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:28:57.640908Z","iopub.execute_input":"2022-07-14T23:28:57.641261Z","iopub.status.idle":"2022-07-14T23:28:57.674784Z","shell.execute_reply.started":"2022-07-14T23:28:57.641232Z","shell.execute_reply":"2022-07-14T23:28:57.673982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.2.3 One-Hot Encoding","metadata":{}},{"cell_type":"markdown","source":"Next we will encode the nominal variable. For this type of variable, we will use one-hot encoding. With one-hot encoding, we convert each categorical value into a new categorical column and assign a binary value of 1 or 0 to those columns. We will only use this method on variables with a number of unique values less than 6.","metadata":{}},{"cell_type":"code","source":"cat_var = cat_var.drop(ord_all)\nonehot_var = cat_var[cat_var < 6].index\n\ndf_train = pd.get_dummies(df_train, prefix=onehot_var, columns=onehot_var)\ndf_test = pd.get_dummies(df_test, prefix=onehot_var, columns=onehot_var)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:29:01.764713Z","iopub.execute_input":"2022-07-14T23:29:01.765676Z","iopub.status.idle":"2022-07-14T23:29:01.813622Z","shell.execute_reply.started":"2022-07-14T23:29:01.765630Z","shell.execute_reply":"2022-07-14T23:29:01.812407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get encoded variables name that do not yet exist in the test data\nadd_var = [var for var in df_train.columns if var not in df_test.columns]\n\n# Add new columns in the test data with value of 0\nfor var in add_var:\n    if var != \"SalePrice\":\n        df_test[var] = 0\n\n# Reorder test data column so it is the same order as the train data\ndf_test = df_test[df_train.columns.drop(\"SalePrice\")]","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:29:05.328941Z","iopub.execute_input":"2022-07-14T23:29:05.329310Z","iopub.status.idle":"2022-07-14T23:29:05.340717Z","shell.execute_reply.started":"2022-07-14T23:29:05.329281Z","shell.execute_reply":"2022-07-14T23:29:05.339983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.2.4 Target Encoding","metadata":{}},{"cell_type":"markdown","source":"The problem with One-Hot Encoding is that the more unique values in a variable, the more new columns will be created. In that case, it can lead to high memory consumption and increase the computational cost. Therefore, we will use target encoding for variables with 6 or more unique values. \n\nActually, we missed something before. There is a variable which is actually categorical, but that variable in this dataset has a numerical value (int). That variable is **MoSold**, which contains the value of what month the house was sold. So, we will add that variable when we do the target encoding","metadata":{}},{"cell_type":"code","source":"cat_var = cat_var.drop(onehot_var)\nX_train = df_train.drop(\"SalePrice\", axis=1)\ny_train = df_train[\"SalePrice\"]\n\nte = MEstimateEncoder(cols=df_train[cat_var.index.append(pd.Index([\"MoSold\"]))]) # Add MoSold variable to the encoder\nX_train = te.fit_transform(X_train, y_train)\ndf_test = te.transform(df_test)\n\ndf_train = pd.concat([X_train, y_train], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T23:29:59.285645Z","iopub.execute_input":"2022-07-14T23:29:59.286051Z","iopub.status.idle":"2022-07-14T23:29:59.517644Z","shell.execute_reply.started":"2022-07-14T23:29:59.286018Z","shell.execute_reply":"2022-07-14T23:29:59.516519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Creating and Deleting Variables","metadata":{}},{"cell_type":"markdown","source":"In this process, we will create some new variables from existing data. We need to do this mainly because we don't want any multicollinearity in our data. Multicollinearity happens when independent variables in the regression model are highly correlated to each other. It makes it hard to interpret of model and also creates an overfitting problem. Another reason we want to do this is because we can create new features that might be useful for predicting targets.","metadata":{}},{"cell_type":"markdown","source":"#### 2.3.1 Checking Correlation","metadata":{}},{"cell_type":"markdown","source":"Let's check the correlation between variables in our data first.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,12))\nsns.heatmap(df_train[num_var].drop(\"Id\", axis=1).corr())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:01:11.875415Z","iopub.execute_input":"2022-07-15T00:01:11.875788Z","iopub.status.idle":"2022-07-15T00:01:12.868867Z","shell.execute_reply.started":"2022-07-15T00:01:11.875758Z","shell.execute_reply":"2022-07-15T00:01:12.867703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, some variables are highly correlated with other variables. Brighter color means they have positive correlation, meaning both variables move in the same direction, while darker color means they have negative correlation, meaning that when one variable's value increases, the other variables' values decrease.\n\nWe can't clearly see the correlation in the graph above because there are too many variables, so let's sort by the highest correlation. We will only display variables with more than 0.5 correlation coefficient (including negative correlation).","metadata":{}},{"cell_type":"code","source":"df_train_corr = df_train[num_var].corr().abs().unstack().sort_values(kind=\"quicksort\", ascending=False).reset_index()\ndf_train_corr.rename(columns={\"level_0\": \"Feature 1\", \"level_1\": \"Feature 2\", 0: 'Correlation Coefficient'}, inplace=True)\ndf_train_corr.drop(df_train_corr.iloc[1::2].index, inplace=True)\ndf_train_corr = df_train_corr.drop(df_train_corr[df_train_corr['Correlation Coefficient'] == 1.0].index)\n\nhigh_corr = df_train_corr['Correlation Coefficient'] > 0.5\ndf_train_corr[high_corr].reset_index(drop=True).style.background_gradient('summer_r')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:20:04.972230Z","iopub.execute_input":"2022-07-15T00:20:04.972752Z","iopub.status.idle":"2022-07-15T00:20:05.019311Z","shell.execute_reply.started":"2022-07-15T00:20:04.972697Z","shell.execute_reply":"2022-07-15T00:20:05.018535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There we go. Now let's also check which variables have a high correlation with the target.","metadata":{}},{"cell_type":"code","source":"df_train_corr[high_corr].loc[(df_train_corr[\"Feature 1\"]==\"SalePrice\") | (df_train_corr[\"Feature 2\"]==\"SalePrice\")].reset_index(drop=True).style.background_gradient('summer_r')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T00:25:18.688792Z","iopub.execute_input":"2022-07-15T00:25:18.689846Z","iopub.status.idle":"2022-07-15T00:25:18.709168Z","shell.execute_reply.started":"2022-07-15T00:25:18.689805Z","shell.execute_reply":"2022-07-15T00:25:18.708179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.3.2 Creating New Variables","metadata":{}},{"cell_type":"markdown","source":"Now we will create some new variables based on variables that have a high correlation in the data above to avoid collinearity. We will also create new features that might be useful.","metadata":{}},{"cell_type":"code","source":"for df in [df_train, df_test]:\n    df[\"GarAreaPerCar\"] = (df[\"GarageArea\"] / df[\"GarageCars\"]).fillna(0)\n    df[\"GrLivAreaPerRoom\"] = df[\"GrLivArea\"] / df[\"TotRmsAbvGrd\"]\n    df[\"TotalHouseSF\"] = df[\"TotalBsmtSF\"] + df[\"1stFlrSF\"] + df[\"2ndFlrSF\"]\n    df[\"TotalFullBath\"] = df[\"FullBath\"] + df[\"BsmtFullBath\"]\n    df[\"TotalHalfBath\"] = df[\"HalfBath\"] + df[\"BsmtHalfBath\"]\n    df[\"InitHouseAge\"] = df[\"YrSold\"] - df[\"YearBuilt\"]\n    df[\"RemodHouseAge\"] = df[\"InitHouseAge\"] - (df[\"YrSold\"] - df[\"YearRemodAdd\"])\n    df[\"IsRemod\"] = (df[\"YearRemodAdd\"] - df[\"YearBuilt\"]).apply(lambda x: 1 if x > 0 else 0)\n    df[\"GarageAge\"] = (df[\"YrSold\"] - df[\"GarageYrBlt\"]).apply(lambda x: 0 if x > 2000 else x)\n    df[\"IsGarage\"] = df[\"GarageYrBlt\"].apply(lambda x: 1 if x > 0 else 0)\n    df['TotalPorchSF'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['3SsnPorch'] + df['ScreenPorch']\n    df[\"AvgQualCond\"] = (df[\"OverallQual\"] + df[\"OverallCond\"]) / 2","metadata":{"execution":{"iopub.status.busy":"2022-07-13T01:51:59.546511Z","iopub.execute_input":"2022-07-13T01:51:59.547133Z","iopub.status.idle":"2022-07-13T01:51:59.588448Z","shell.execute_reply.started":"2022-07-13T01:51:59.547095Z","shell.execute_reply":"2022-07-13T01:51:59.587412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.3.3 Deleting Variables","metadata":{}},{"cell_type":"markdown","source":"We are going to remove some variables since we no longer need them. This will also reduce memory consumption because the features we will use are also reduced.","metadata":{}},{"cell_type":"code","source":"for df in [df_train, df_test]:\n    df.drop([\n        \"GarageArea\", \"GarageCars\", \"GrLivArea\", \n        \"TotRmsAbvGrd\", \"TotalBsmtSF\", \"1stFlrSF\", \n        \"2ndFlrSF\", \"FullBath\", \"BsmtFullBath\", \"HalfBath\", \n        \"BsmtHalfBath\", \"YrSold\", \"YearBuilt\", \"YearRemodAdd\",\n        \"GarageYrBlt\", \"OpenPorchSF\", \"EnclosedPorch\", \"3SsnPorch\",\n        \"ScreenPorch\", \"OverallQual\", \"OverallCond\"\n    ], axis=1, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Model Building","metadata":{}},{"cell_type":"markdown","source":"#### 3.1 Splitting Dataset","metadata":{}},{"cell_type":"code","source":"X_train = df_train.drop([\"Id\", \"SalePrice\"], axis=1)\ny_train = df_train.SalePrice\n\nX_test = df_test.drop(\"Id\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:15:00.395453Z","iopub.execute_input":"2022-07-15T01:15:00.395879Z","iopub.status.idle":"2022-07-15T01:15:00.406186Z","shell.execute_reply.started":"2022-07-15T01:15:00.395833Z","shell.execute_reply":"2022-07-15T01:15:00.404916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.2 Feature Scaling","metadata":{}},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\ny_train_log = np.log10(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:15:19.077701Z","iopub.execute_input":"2022-07-15T01:15:19.078082Z","iopub.status.idle":"2022-07-15T01:15:19.097872Z","shell.execute_reply.started":"2022-07-15T01:15:19.078051Z","shell.execute_reply":"2022-07-15T01:15:19.096819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.3 Selecting Best Model","metadata":{}},{"cell_type":"code","source":"regressors = {\n    \"XGB Regressor\": XGBRegressor(),\n    \"LGBM Regressor\": LGBMRegressor(),\n    \"Lasso\": Lasso(),\n    \"Ridge\": Ridge(),\n    \"Elastic Net\": ElasticNet(),\n    \"Bayesian Ridge\": BayesianRidge(),\n    \"SVR\": SVR(),\n    \"GB Regressor\": GradientBoostingRegressor(random_state=0)\n}\n\nresults = pd.DataFrame(columns=[\"Regressor\", \"Avg_RMSE\"])\nfor name, reg in regressors.items():\n    model = reg\n    cv_results = cross_validate(\n        model, X_train_scaled, y_train_log, cv=10,\n        scoring=(['neg_root_mean_squared_error'])\n    )\n\n    results = results.append({\n        \"Regressor\": name,\n        \"Avg_RMSE\": np.abs(cv_results['test_neg_root_mean_squared_error']).mean()\n    }, ignore_index=True)\n\nresults = results.sort_values(\"Avg_RMSE\", ascending=True)\nresults.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:15:27.506981Z","iopub.execute_input":"2022-07-15T01:15:27.507371Z","iopub.status.idle":"2022-07-15T01:15:47.135012Z","shell.execute_reply.started":"2022-07-15T01:15:27.507341Z","shell.execute_reply":"2022-07-15T01:15:47.133680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.barplot(data=results, x=\"Avg_RMSE\", y=\"Regressor\")\nplt.title(\"Average RMSE CV Score\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:18:43.077356Z","iopub.execute_input":"2022-07-15T01:18:43.077759Z","iopub.status.idle":"2022-07-15T01:18:43.362567Z","shell.execute_reply.started":"2022-07-15T01:18:43.077725Z","shell.execute_reply":"2022-07-15T01:18:43.361269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will use 3 best algorithms from the cross validation test above, which is GB Regressor, LGBM Regressor, and XGB Regressor. Let's tune their hyperparameters using Grid Search Cross Validation.","metadata":{}},{"cell_type":"markdown","source":"#### 3.4 Hyperparameter Tuning","metadata":{}},{"cell_type":"markdown","source":"#### Gradient Boosting Regressor","metadata":{}},{"cell_type":"code","source":"gbr = GradientBoostingRegressor(random_state=0)\nparams = {\n    \"loss\": (\"squared_error\", \"absolute_error\"),\n    \"learning_rate\": (1.0, 0.1, 0.01),\n    \"n_estimators\": (50, 100, 200)\n}\nreg1 = GridSearchCV(gbr, params, cv=10)\nreg1.fit(X_train_scaled, y_train_log)\nprint(\"Best hyperparameter:\", reg1.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:27:25.739373Z","iopub.execute_input":"2022-07-15T01:27:25.740191Z","iopub.status.idle":"2022-07-15T01:30:33.413120Z","shell.execute_reply.started":"2022-07-15T01:27:25.740145Z","shell.execute_reply":"2022-07-15T01:30:33.411947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = reg1.predict(X_train_scaled)\nprint(f\"Train RMSE: {mean_squared_error(y_train_log, y_pred, squared=False)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:30:33.415065Z","iopub.execute_input":"2022-07-15T01:30:33.415405Z","iopub.status.idle":"2022-07-15T01:30:33.428669Z","shell.execute_reply.started":"2022-07-15T01:30:33.415375Z","shell.execute_reply":"2022-07-15T01:30:33.427267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### XGB Regressor","metadata":{}},{"cell_type":"code","source":"xgb = XGBRegressor(random_state=0)\nparams = {\n    \"max_depth\": (3, 6, 9),\n    \"learning_rate\": (0.3, 0.1, 0.05),\n    \"n_estimators\": (50, 100, 200)\n}\nreg2 = GridSearchCV(xgb, params, cv=10)\nreg2.fit(X_train_scaled, y_train_log)\nprint(\"Best hyperparameter:\", reg2.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:30:33.430368Z","iopub.execute_input":"2022-07-15T01:30:33.431271Z","iopub.status.idle":"2022-07-15T01:34:29.987905Z","shell.execute_reply.started":"2022-07-15T01:30:33.431235Z","shell.execute_reply":"2022-07-15T01:34:29.986924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = reg2.predict(X_train_scaled)\nprint(f\"Train RMSE: {mean_squared_error(y_train_log, y_pred, squared=False)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:34:29.992972Z","iopub.execute_input":"2022-07-15T01:34:29.995176Z","iopub.status.idle":"2022-07-15T01:34:30.012027Z","shell.execute_reply.started":"2022-07-15T01:34:29.995134Z","shell.execute_reply":"2022-07-15T01:34:30.011075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LGBM Regressor","metadata":{}},{"cell_type":"code","source":"lgbm = LGBMRegressor(random_state=0)\nparams = {\n    \"num_leaves\": (11, 31, 51),\n    \"learning_rate\": (0.5, 0.1, 0.05),\n    \"n_estimators\": (50, 100, 200)\n}\nreg3 = GridSearchCV(lgbm, params, cv=10)\nreg3.fit(X_train_scaled, y_train_log)\nprint(\"Best hyperparameter:\", reg3.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:34:30.016103Z","iopub.execute_input":"2022-07-15T01:34:30.018227Z","iopub.status.idle":"2022-07-15T01:35:38.141725Z","shell.execute_reply.started":"2022-07-15T01:34:30.018181Z","shell.execute_reply":"2022-07-15T01:35:38.140886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = reg3.predict(X_train_scaled)\nprint(f\"Train RMSE: {mean_squared_error(y_train_log, y_pred, squared=False)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T01:35:38.143105Z","iopub.execute_input":"2022-07-15T01:35:38.143749Z","iopub.status.idle":"2022-07-15T01:35:38.162162Z","shell.execute_reply.started":"2022-07-15T01:35:38.143713Z","shell.execute_reply":"2022-07-15T01:35:38.161216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3.5 Stacking 3 Regressors","metadata":{}},{"cell_type":"code","source":"def sm_predict(X):\n    return (3 * reg1.predict(X) + 2 * reg2.predict(X) + 5 * reg3.predict(X)) / 10\n\ny_pred_stack = sm_predict(X_train_scaled)\nprint(f\"Train RMSE with Stacking: {mean_squared_error(y_train_log, y_pred_stack, squared=False)}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T02:03:21.577129Z","iopub.execute_input":"2022-07-15T02:03:21.577573Z","iopub.status.idle":"2022-07-15T02:03:21.645753Z","shell.execute_reply.started":"2022-07-15T02:03:21.577535Z","shell.execute_reply":"2022-07-15T02:03:21.644873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Submission","metadata":{}},{"cell_type":"code","source":"y_pred = sm_predict(X_test_scaled)\ny_pred_inv = 10 ** y_pred\n\nsubmission = pd.DataFrame({'Id': df_test.Id, 'SalePrice': y_pred_inv})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T02:04:32.123590Z","iopub.execute_input":"2022-07-15T02:04:32.123985Z","iopub.status.idle":"2022-07-15T02:04:32.186124Z","shell.execute_reply.started":"2022-07-15T02:04:32.123954Z","shell.execute_reply":"2022-07-15T02:04:32.184919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}