{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook has been inspired from some of the great notebooks that I've read. This notebook contains some code that have been reproduced directly from these notebooks\n\n[Comprehensive data exploration with Python by Pedro Marcelino](https://www.kaggle.com/code/pmarcelino/comprehensive-data-exploration-with-python/notebook) \n\n[A study on Regression applied to the Ames dataset by Julien Cohen-Solal](https://www.kaggle.com/code/juliencs/a-study-on-regression-applied-to-the-ames-dataset/notebook)\n\n[Regularized Linear Models by Alexandru Papiu](https://www.kaggle.com/code/apapiu/regularized-linear-models/notebook)\n\n[Stacked Regressions : Top 4% on LeaderBoard by Serigne](https://www.kaggle.com/code/serigne/stacked-regressions-top-4-on-leaderboard)","metadata":{}},{"cell_type":"code","source":"#importing basic libraries\nimport numpy as np\nimport pandas as pd\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n#for plots\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#preprocessing & feature engineering\nfrom scipy import stats\nfrom scipy.stats import norm, skew\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.preprocessing import LabelEncoder\nfrom category_encoders import MEstimateEncoder\nfrom sklearn.model_selection import KFold\nfrom functools import reduce\n\n\n#modelling\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\nfrom sklearn.linear_model import ElasticNet, Lasso\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics import silhouette_score\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import GridSearchCV","metadata":{"papermill":{"duration":2.049208,"end_time":"2022-07-19T06:51:16.747699","exception":false,"start_time":"2022-07-19T06:51:14.698491","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-25T03:36:24.447662Z","iopub.execute_input":"2022-07-25T03:36:24.448161Z","iopub.status.idle":"2022-07-25T03:36:27.613064Z","shell.execute_reply.started":"2022-07-25T03:36:24.448044Z","shell.execute_reply":"2022-07-25T03:36:27.611831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading data from csv\ndata = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv', index_col=0)\ntest = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv', index_col=0)\n\n#making a copy of original data for EDA\ntrain = data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:05.179225Z","iopub.execute_input":"2022-07-21T09:26:05.179542Z","iopub.status.idle":"2022-07-21T09:26:05.267025Z","shell.execute_reply.started":"2022-07-21T09:26:05.179514Z","shell.execute_reply":"2022-07-21T09:26:05.266003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step1 - Visualization**","metadata":{"papermill":{"duration":0.055119,"end_time":"2022-07-19T06:51:17.062972","exception":false,"start_time":"2022-07-19T06:51:17.007853","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#feature set\ntrain.columns","metadata":{"papermill":{"duration":0.068639,"end_time":"2022-07-19T06:51:17.185654","exception":false,"start_time":"2022-07-19T06:51:17.117015","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:05.268598Z","iopub.execute_input":"2022-07-21T09:26:05.269941Z","iopub.status.idle":"2022-07-21T09:26:05.278865Z","shell.execute_reply.started":"2022-07-21T09:26:05.269888Z","shell.execute_reply":"2022-07-21T09:26:05.277554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"papermill":{"duration":0.090566,"end_time":"2022-07-19T06:51:17.331102","exception":false,"start_time":"2022-07-19T06:51:17.240536","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:05.281481Z","iopub.execute_input":"2022-07-21T09:26:05.282101Z","iopub.status.idle":"2022-07-21T09:26:05.316459Z","shell.execute_reply.started":"2022-07-21T09:26:05.282067Z","shell.execute_reply":"2022-07-21T09:26:05.315336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our target variable is SalePrice, so lets visualize it","metadata":{"papermill":{"duration":0.053766,"end_time":"2022-07-19T06:51:17.439700","exception":false,"start_time":"2022-07-19T06:51:17.385934","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Target label distribution**","metadata":{"papermill":{"duration":0.053999,"end_time":"2022-07-19T06:51:17.548602","exception":false,"start_time":"2022-07-19T06:51:17.494603","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#check the distribution of 'SalePrice'\nfig, (ax1,ax2) = plt.subplots(1,2, figsize=(15,6))\n\nsns.histplot(x='SalePrice', data=train, kde=True, ax=ax1)\nax1.set(ylabel = 'frequency')\nax1.set(xlabel = 'SalePrice')\nax1.set(title = 'SalePrice Distribution');\n\n#QQplot\nstats.probplot(train['SalePrice'], plot=ax2)\nax2.set_title(\"QQplot of SalePrice\");","metadata":{"papermill":{"duration":0.556007,"end_time":"2022-07-19T06:51:18.159768","exception":false,"start_time":"2022-07-19T06:51:17.603761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:05.318147Z","iopub.execute_input":"2022-07-21T09:26:05.318466Z","iopub.status.idle":"2022-07-21T09:26:05.803262Z","shell.execute_reply.started":"2022-07-21T09:26:05.318438Z","shell.execute_reply":"2022-07-21T09:26:05.802011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that our target distribution is skewed, to be precise, right-skewed. \nWe shall log transform our target variable","metadata":{"papermill":{"duration":0.055461,"end_time":"2022-07-19T06:51:18.271285","exception":false,"start_time":"2022-07-19T06:51:18.215824","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Relation with numerical features**","metadata":{"papermill":{"duration":0.054697,"end_time":"2022-07-19T06:51:18.380762","exception":false,"start_time":"2022-07-19T06:51:18.326065","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We check relation of our target variable with some of the numerical features","metadata":{"papermill":{"duration":0.054717,"end_time":"2022-07-19T06:51:18.490219","exception":false,"start_time":"2022-07-19T06:51:18.435502","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#scatter plot GrLivArea/SalePrice\nsns.scatterplot(x='GrLivArea', y='SalePrice', data=train)\nplt.title('Living Area vs SalePrice');","metadata":{"papermill":{"duration":0.23651,"end_time":"2022-07-19T06:51:18.782028","exception":false,"start_time":"2022-07-19T06:51:18.545518","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:05.804983Z","iopub.execute_input":"2022-07-21T09:26:05.805440Z","iopub.status.idle":"2022-07-21T09:26:06.035290Z","shell.execute_reply.started":"2022-07-21T09:26:05.805397Z","shell.execute_reply":"2022-07-21T09:26:06.034047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see two outliers straight away on the right corner, these houses with huge living area were sold at very less price.\nWe shall remove this two outliers","metadata":{"papermill":{"duration":0.057051,"end_time":"2022-07-19T06:51:18.896828","exception":false,"start_time":"2022-07-19T06:51:18.839777","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#scatter plot TotalBsmtSF/SalePrice\nsns.scatterplot(x='TotalBsmtSF', y='SalePrice', data=train)\nplt.title('Total Basement Area vs SalePrice');","metadata":{"papermill":{"duration":0.227947,"end_time":"2022-07-19T06:51:19.180467","exception":false,"start_time":"2022-07-19T06:51:18.952520","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:06.036508Z","iopub.execute_input":"2022-07-21T09:26:06.036835Z","iopub.status.idle":"2022-07-21T09:26:06.393000Z","shell.execute_reply.started":"2022-07-21T09:26:06.036807Z","shell.execute_reply":"2022-07-21T09:26:06.391666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here too, we can see an outlier on extreme right of the plot","metadata":{"papermill":{"duration":0.055945,"end_time":"2022-07-19T06:51:19.294004","exception":false,"start_time":"2022-07-19T06:51:19.238059","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#scatter plot LotFrontage/SalePrice\nsns.scatterplot(x='LotFrontage', y='SalePrice', data=train)\nplt.title('LotFrontage vs SalePrice');","metadata":{"papermill":{"duration":0.2238,"end_time":"2022-07-19T06:51:19.574323","exception":false,"start_time":"2022-07-19T06:51:19.350523","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:06.394466Z","iopub.execute_input":"2022-07-21T09:26:06.394811Z","iopub.status.idle":"2022-07-21T09:26:06.612418Z","shell.execute_reply.started":"2022-07-21T09:26:06.394780Z","shell.execute_reply":"2022-07-21T09:26:06.611034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see outliers in most of the numerical features. We shall remove these outliers for better performance of the model","metadata":{"papermill":{"duration":0.056489,"end_time":"2022-07-19T06:51:19.687833","exception":false,"start_time":"2022-07-19T06:51:19.631344","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Relation with Categorical features**","metadata":{"papermill":{"duration":0.056269,"end_time":"2022-07-19T06:51:19.800537","exception":false,"start_time":"2022-07-19T06:51:19.744268","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#box plot MSZoning/SalePrice\nf, ax = plt.subplots(figsize=(10, 6))\nfig = sns.boxplot(x='MSZoning', y='SalePrice', data=train)","metadata":{"papermill":{"duration":0.260656,"end_time":"2022-07-19T06:51:20.122324","exception":false,"start_time":"2022-07-19T06:51:19.861668","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:06.613906Z","iopub.execute_input":"2022-07-21T09:26:06.614275Z","iopub.status.idle":"2022-07-21T09:26:06.868591Z","shell.execute_reply.started":"2022-07-21T09:26:06.614243Z","shell.execute_reply":"2022-07-21T09:26:06.867364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#box plot OverQual/SalePrice\nf, ax = plt.subplots(figsize=(10, 6))\nfig = sns.boxplot(x='OverallQual', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.328877,"end_time":"2022-07-19T06:51:20.509694","exception":false,"start_time":"2022-07-19T06:51:20.180817","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:06.873701Z","iopub.execute_input":"2022-07-21T09:26:06.874049Z","iopub.status.idle":"2022-07-21T09:26:07.202069Z","shell.execute_reply.started":"2022-07-21T09:26:06.874020Z","shell.execute_reply":"2022-07-21T09:26:07.200925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Step2 - Preprocessing**","metadata":{"papermill":{"duration":0.05716,"end_time":"2022-07-19T06:51:20.624374","exception":false,"start_time":"2022-07-19T06:51:20.567214","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Missing Data**","metadata":{"papermill":{"duration":0.058133,"end_time":"2022-07-19T06:51:20.740406","exception":false,"start_time":"2022-07-19T06:51:20.682273","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We check for missing values in our dataset","metadata":{"papermill":{"duration":0.057883,"end_time":"2022-07-19T06:51:20.859037","exception":false,"start_time":"2022-07-19T06:51:20.801154","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#function to give percentage of missing values in each feature\ndef missing_val(X):\n    #percentage of missing values per feature\n    missing = X.isna().sum()/len(train)*100\n    #dropping columns with no missing values\n    missing = missing.drop(missing[missing==0].index).sort_values(ascending=False)\n    #converting to dataframe\n    df_missing = pd.DataFrame({'missing_ratio': missing})\n    return df_missing\n\n#check missing values for training dataset\ndf_train_missing = missing_val(train)\ndf_test_missing = missing_val(test)\ndf_train_missing","metadata":{"papermill":{"duration":0.099862,"end_time":"2022-07-19T06:51:21.016201","exception":false,"start_time":"2022-07-19T06:51:20.916339","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:07.203299Z","iopub.execute_input":"2022-07-21T09:26:07.203635Z","iopub.status.idle":"2022-07-21T09:26:07.241760Z","shell.execute_reply.started":"2022-07-21T09:26:07.203605Z","shell.execute_reply":"2022-07-21T09:26:07.240660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nfig,(ax1, ax2) = plt.subplots(2,1,figsize=(13,12))\nsns.barplot(y=df_train_missing.index, x='missing_ratio', data=df_train_missing, ax=ax1)\nax1.set_title('Percentage of missing values in training data')\nax1.bar_label(ax1.containers[0], fmt='%.2f'); #annotations\n\nsns.barplot(y=df_test_missing.index, x='missing_ratio', data=df_test_missing, ax=ax2)\nax2.set_title('Percentage of missing values in test data');\nax2.bar_label(ax2.containers[0], fmt='%.2f'); #annotations","metadata":{"papermill":{"duration":1.147351,"end_time":"2022-07-19T06:51:22.285682","exception":false,"start_time":"2022-07-19T06:51:21.138331","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:07.243259Z","iopub.execute_input":"2022-07-21T09:26:07.243688Z","iopub.status.idle":"2022-07-21T09:26:08.236171Z","shell.execute_reply.started":"2022-07-21T09:26:07.243653Z","shell.execute_reply":"2022-07-21T09:26:08.234944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How we handle this missing data will be critical, since missing values in each feature convey, a different meaning.\nE.g : Missing value in PoolQC probably means there is no pool","metadata":{"papermill":{"duration":0.058743,"end_time":"2022-07-19T06:51:22.403309","exception":false,"start_time":"2022-07-19T06:51:22.344566","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We create a function to fill in the missing values of feature. We read the description file to get an idea as to what a missing value means in each feature ","metadata":{"papermill":{"duration":0.058392,"end_time":"2022-07-19T06:51:22.520929","exception":false,"start_time":"2022-07-19T06:51:22.462537","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#we create a dict mapping feature with its replacement for a missing value in the feature \nimpute_dict = {}\n#Alley: NA means alley access\nimpute_dict['Alley'] = 'No'\n#LotShape: NA would most likely mean Regular\nimpute_dict['LotShape'] = 'Reg'\n#Utilities: NA would most likely mean AllUtilities\nimpute_dict['Utilities'] = 'AllPub'\n#MasVnrType: NA would probably mean no veneer\nimpute_dict['MasVnrType'] = 'None'\nimpute_dict['MasVnrArea'] = 0\n#BsmtQual: NA means no basement available\nimpute_dict['BsmtQual'] = 'No'\nimpute_dict['BsmtCond'] = 'No'\nimpute_dict['BsmtExposure'] = 'No'\nimpute_dict['BsmtFinType1'] = 'No'\nimpute_dict['BsmtFinType2'] = 'No'\nimpute_dict['BsmtFullBath'] = 0\nimpute_dict['BsmtHalfBath'] = 0\nimpute_dict['BsmtUnfSF'] = 0\nimpute_dict['BsmtFinSF1'] = 0\nimpute_dict['BsmtFinSF2'] = 0\nimpute_dict['TotalBsmtSF'] = 0\n#HeatingQC: NA would most likely mean Typical\nimpute_dict['HeatingQC'] = 'TA'\n#CentralAir: NA would most likely mean No central air conditioning\nimpute_dict['CentralAir'] = 'No'\n#2ndFlrSF: NA would most likely mean no 2nd floor present\nimpute_dict['2ndFlrSF'] = 0\n#HalfBath: NA would most likely mean no half baths present\nimpute_dict['HalfBath'] = 0\n#FullBath: NA would most likely mean no full baths present\nimpute_dict['FullBath'] = 0\n#Bedroom: NA would most likely mean 0\nimpute_dict['BedroomAbvGr'] = 0\n#KitchenQual: NA would most likely mean Typical\nimpute_dict['KitchenQual'] = 'TA'\n#Functional: NA would most likely meean Typical\nimpute_dict['Functional'] = 'Typ'\n#Fireplaces: NA would most likely mean no fireplaces available\nimpute_dict['Fireplaces'] = 0\nimpute_dict['FireplaceQu'] = 'No'\n#GarageType: NA would mean absence of a garage\nimpute_dict['GarageType'] = 'No'\nimpute_dict['GarageFinish'] = 'No'\nimpute_dict['GarageCars'] = 0\nimpute_dict['GarageArea'] = 0\nimpute_dict['GarageQual'] = 'No'\nimpute_dict['GarageCond'] = 'No'\nimpute_dict['GarageYrBlt'] = 0\n#PoolArea: NA would most likley mean absence of a pool\nimpute_dict['PoolArea'] = 0\nimpute_dict['PoolQC'] = 'No'\n#Fence: NA would mean no fence available\nimpute_dict['Fence'] = 'No'\n#MiscFeature: NA would mean no additional feature\nimpute_dict['MiscFeature'] = 'No'\n#SaleCondition: NA would most likely mean normal sale\nimpute_dict['SaleCondition'] = 'Normal'","metadata":{"papermill":{"duration":0.076709,"end_time":"2022-07-19T06:51:22.656220","exception":false,"start_time":"2022-07-19T06:51:22.579511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:08.238012Z","iopub.execute_input":"2022-07-21T09:26:08.238465Z","iopub.status.idle":"2022-07-21T09:26:08.254169Z","shell.execute_reply.started":"2022-07-21T09:26:08.238421Z","shell.execute_reply":"2022-07-21T09:26:08.252938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are some other features in which we impute missing values by the use of central tendencies of that feature","metadata":{"papermill":{"duration":0.062085,"end_time":"2022-07-19T06:51:22.779118","exception":false,"start_time":"2022-07-19T06:51:22.717033","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **Lot Frontage**","metadata":{"papermill":{"duration":0.059151,"end_time":"2022-07-19T06:51:22.897902","exception":false,"start_time":"2022-07-19T06:51:22.838751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#subplots\nfig,axs = plt.subplots(5,5, figsize=(20,12))\naxs = axs.ravel()\n#lot frontage kde plot for each neighborhood\nfor neigh, ax in zip(train['Neighborhood'].unique(), axs):\n    fig.suptitle('Density plot of LotFrontage in each neighborhood',fontweight =\"bold\")\n    sns.kdeplot(train[train['Neighborhood'] == neigh]['LotFrontage'], ax=ax)\n    #title of subplot\n    ax.set_title(neigh)\n    #setting X-axis and labels to false\n    #ax.xaxis.label.set_visible(False)\n    #ax.set_xticks([])\n    #setting Y-aix labels to false\n    #ax.set_yticks([])\n    #ax.yaxis.label.set_visible(False);\n    \nplt.tight_layout()","metadata":{"papermill":{"duration":3.120201,"end_time":"2022-07-19T06:51:26.076644","exception":false,"start_time":"2022-07-19T06:51:22.956443","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:08.255968Z","iopub.execute_input":"2022-07-21T09:26:08.256847Z","iopub.status.idle":"2022-07-21T09:26:11.674664Z","shell.execute_reply.started":"2022-07-21T09:26:08.256802Z","shell.execute_reply":"2022-07-21T09:26:11.673729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LotFrontage of a property will most likely be similar to lot frontage size of other houses in the neighborhood.\nWe fill in the median value","metadata":{"papermill":{"duration":0.062279,"end_time":"2022-07-19T06:51:26.202036","exception":false,"start_time":"2022-07-19T06:51:26.139757","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **Electrical**","metadata":{"papermill":{"duration":0.061699,"end_time":"2022-07-19T06:51:26.326097","exception":false,"start_time":"2022-07-19T06:51:26.264398","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train['Electrical'].value_counts().plot(kind='pie', autopct='%.2f', radius=1.5);","metadata":{"papermill":{"duration":0.352825,"end_time":"2022-07-19T06:51:26.741149","exception":false,"start_time":"2022-07-19T06:51:26.388324","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:11.675831Z","iopub.execute_input":"2022-07-21T09:26:11.676829Z","iopub.status.idle":"2022-07-21T09:26:11.825029Z","shell.execute_reply.started":"2022-07-21T09:26:11.676785Z","shell.execute_reply":"2022-07-21T09:26:11.823561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dominated by one type, we fill the most common value in the dataset","metadata":{"papermill":{"duration":0.062599,"end_time":"2022-07-19T06:51:26.904153","exception":false,"start_time":"2022-07-19T06:51:26.841554","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **MSZoning**","metadata":{"papermill":{"duration":0.062361,"end_time":"2022-07-19T06:51:27.028923","exception":false,"start_time":"2022-07-19T06:51:26.966562","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(16,6))\nsns.countplot(x='Neighborhood', hue='MSZoning', data=train, ax=ax)\nplt.xticks(rotation=60);","metadata":{"papermill":{"duration":0.738102,"end_time":"2022-07-19T06:51:27.830173","exception":false,"start_time":"2022-07-19T06:51:27.092071","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:11.827280Z","iopub.execute_input":"2022-07-21T09:26:11.827890Z","iopub.status.idle":"2022-07-21T09:26:12.542971Z","shell.execute_reply.started":"2022-07-21T09:26:11.827837Z","shell.execute_reply":"2022-07-21T09:26:12.541675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the houses of a neighborhood will belong to particular MS Zone. We fill in the most commom value","metadata":{"papermill":{"duration":0.062258,"end_time":"2022-07-19T06:51:27.955673","exception":false,"start_time":"2022-07-19T06:51:27.893415","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **Exterior1st & Exterior2nd**","metadata":{"papermill":{"duration":0.063459,"end_time":"2022-07-19T06:51:28.083525","exception":false,"start_time":"2022-07-19T06:51:28.020066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig,(ax1,ax2) = plt.subplots(1,2,figsize=(16,6))\ntrain['Exterior1st'].value_counts().plot(kind='pie', autopct='%.2f', ax=ax1)\nax1.set(title='Exterior1st')\ntrain['Exterior2nd'].value_counts().plot(kind='pie', autopct='%.2f',  ax=ax2)\nax2.set(title='Exterior2nd');","metadata":{"papermill":{"duration":0.592908,"end_time":"2022-07-19T06:51:28.739879","exception":false,"start_time":"2022-07-19T06:51:28.146971","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:12.544135Z","iopub.execute_input":"2022-07-21T09:26:12.544463Z","iopub.status.idle":"2022-07-21T09:26:13.092397Z","shell.execute_reply.started":"2022-07-21T09:26:12.544434Z","shell.execute_reply":"2022-07-21T09:26:13.091347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will the most common value in the dataset","metadata":{"papermill":{"duration":0.064138,"end_time":"2022-07-19T06:51:28.868134","exception":false,"start_time":"2022-07-19T06:51:28.803996","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **SaleType**","metadata":{"papermill":{"duration":0.063846,"end_time":"2022-07-19T06:51:28.998316","exception":false,"start_time":"2022-07-19T06:51:28.934470","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(train['SaleType'].value_counts())\ntrain['SaleType'].value_counts().plot(kind='pie', autopct='%.2f', radius=1.5);","metadata":{"papermill":{"duration":0.24832,"end_time":"2022-07-19T06:51:29.310718","exception":false,"start_time":"2022-07-19T06:51:29.062398","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.093531Z","iopub.execute_input":"2022-07-21T09:26:13.094355Z","iopub.status.idle":"2022-07-21T09:26:13.281467Z","shell.execute_reply.started":"2022-07-21T09:26:13.094318Z","shell.execute_reply":"2022-07-21T09:26:13.279751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dominated by one type of sale, thus we will the most common value in the dataset","metadata":{"papermill":{"duration":0.0652,"end_time":"2022-07-19T06:51:29.466491","exception":false,"start_time":"2022-07-19T06:51:29.401291","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#function to impute missing values in some features based on description of feature\ndef missing_imputer(X, train_data):\n    for col in impute_dict.keys():\n        X.loc[:, col] = X.loc[:, col].fillna(impute_dict[col])\n    \n    #we find central tendencies for training data and use them for imputing values in both training and test data\n    #for lot frontage\n    median_lotfrontage = train_data.groupby(['Neighborhood'])['LotFrontage'].agg('median').to_dict()\n    X['LotFrontage'] = X['LotFrontage'].fillna(X['Neighborhood'].map(median_lotfrontage))\n   \n    \n    #for MS Zoning\n    mszone = train_data.groupby(['Neighborhood'])['MSZoning'].agg(pd.Series.mode).to_dict()\n    X['MSZoning'] = X['MSZoning'].fillna(X['Neighborhood'].map(mszone))\n    \n    #for electrical\n    X['Electrical'] = train_data['Electrical'].fillna(train_data['Electrical'].mode()[0])\n    #for sale type\n    X['SaleType'] = train_data['SaleType'].fillna(train_data['SaleType'].mode()[0])\n    #for exterior1st and exterior2nd\n    X['Exterior1st'] = train_data['Exterior1st'].fillna(train_data['Exterior1st'].mode()[0])\n    X['Exterior2nd'] = train_data['Exterior2nd'].fillna(train_data['Exterior2nd'].mode()[0])\n    \n    return X","metadata":{"papermill":{"duration":0.07892,"end_time":"2022-07-19T06:51:29.609963","exception":false,"start_time":"2022-07-19T06:51:29.531043","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.283951Z","iopub.execute_input":"2022-07-21T09:26:13.285179Z","iopub.status.idle":"2022-07-21T09:26:13.306934Z","shell.execute_reply.started":"2022-07-21T09:26:13.285116Z","shell.execute_reply":"2022-07-21T09:26:13.305276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = missing_imputer(train, train_data=train)","metadata":{"papermill":{"duration":0.110224,"end_time":"2022-07-19T06:51:29.784431","exception":false,"start_time":"2022-07-19T06:51:29.674207","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.308910Z","iopub.execute_input":"2022-07-21T09:26:13.310766Z","iopub.status.idle":"2022-07-21T09:26:13.368927Z","shell.execute_reply.started":"2022-07-21T09:26:13.310687Z","shell.execute_reply":"2022-07-21T09:26:13.367835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#missing values after imputation\nmissing_val(train)","metadata":{"papermill":{"duration":0.088949,"end_time":"2022-07-19T06:51:29.938041","exception":false,"start_time":"2022-07-19T06:51:29.849092","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.370810Z","iopub.execute_input":"2022-07-21T09:26:13.371283Z","iopub.status.idle":"2022-07-21T09:26:13.393369Z","shell.execute_reply.started":"2022-07-21T09:26:13.371238Z","shell.execute_reply":"2022-07-21T09:26:13.392195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No missing values for training dataset!!!","metadata":{"papermill":{"duration":0.064427,"end_time":"2022-07-19T06:51:30.066878","exception":false,"start_time":"2022-07-19T06:51:30.002451","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Encoding Categorical Features**","metadata":{"papermill":{"duration":0.065014,"end_time":"2022-07-19T06:51:30.197103","exception":false,"start_time":"2022-07-19T06:51:30.132089","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Since many of the features in our dataset have ordinality associated with them. We assign them a score ","metadata":{"papermill":{"duration":0.064287,"end_time":"2022-07-19T06:51:30.326141","exception":false,"start_time":"2022-07-19T06:51:30.261854","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We see that scoring of most of the categorical features are done on same basis\n\n    - Ex\n    - Gd\n    - TA\n    - Fa\n    - Po\n    - No\n\nSince there is inherent order in these categories, we will use ordinal encoder, such that they are ranked in the following way\n\n    - Ex\t5\n    - Gd\t4\n    - TA\t3\n    - Fa\t2\n    - Po\t1\n    - No\t0","metadata":{"papermill":{"duration":0.064424,"end_time":"2022-07-19T06:51:30.454935","exception":false,"start_time":"2022-07-19T06:51:30.390511","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We make a list of ordinal features, so that we can map each categorical feature with this ordered categories","metadata":{"papermill":{"duration":0.065986,"end_time":"2022-07-19T06:51:30.585085","exception":false,"start_time":"2022-07-19T06:51:30.519099","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- **Electrical**","metadata":{"papermill":{"duration":0.065557,"end_time":"2022-07-19T06:51:30.715809","exception":false,"start_time":"2022-07-19T06:51:30.650252","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Electrical\n\n       SBrkr\tStandard Circuit Breakers & Romex\n       FuseA\tFuse Box over 60 AMP and all Romex wiring (Average)\t\n       FuseF\t60 AMP Fuse Box and mostly Romex wiring (Fair)\n       FuseP\t60 AMP Fuse Box and mostly knob & tube wiring (poor)\n       Mix\tMixed\n       \nWe can see that there is an order to type of electrial circuits used.\nWe shall confirm it using a plot","metadata":{"papermill":{"duration":0.063854,"end_time":"2022-07-19T06:51:30.848703","exception":false,"start_time":"2022-07-19T06:51:30.784849","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sns.barplot(x='Electrical', y='SalePrice', data=train, order=['SBrkr', 'FuseA', 'FuseF', 'FuseP', 'Mix'], ci=None);","metadata":{"papermill":{"duration":0.251478,"end_time":"2022-07-19T06:51:31.164419","exception":false,"start_time":"2022-07-19T06:51:30.912941","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.395499Z","iopub.execute_input":"2022-07-21T09:26:13.396283Z","iopub.status.idle":"2022-07-21T09:26:13.543230Z","shell.execute_reply.started":"2022-07-21T09:26:13.396234Z","shell.execute_reply":"2022-07-21T09:26:13.542403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['GarageType'].value_counts())\ntrain['GarageType'].value_counts().plot(kind='pie', autopct='%.2f', radius=1.5);","metadata":{"papermill":{"duration":0.227978,"end_time":"2022-07-19T06:51:31.458229","exception":false,"start_time":"2022-07-19T06:51:31.230251","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.544878Z","iopub.execute_input":"2022-07-21T09:26:13.546001Z","iopub.status.idle":"2022-07-21T09:26:13.685366Z","shell.execute_reply.started":"2022-07-21T09:26:13.545953Z","shell.execute_reply":"2022-07-21T09:26:13.684121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,(ax1,ax2) = plt.subplots(1,2,figsize=(16,6))\nsns.barplot(x='GarageType', y='SalePrice', data=train, ax=ax1, ci=None, \n           order=['No', 'CarPort', 'Detchd', '2Types', 'Basment', 'Attchd', 'BuiltIn'])\nsns.scatterplot(x='GarageArea', hue='GarageType', y='SalePrice', data=train, ax=ax2);","metadata":{"papermill":{"duration":0.627207,"end_time":"2022-07-19T06:51:32.177731","exception":false,"start_time":"2022-07-19T06:51:31.550524","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:13.687228Z","iopub.execute_input":"2022-07-21T09:26:13.687996Z","iopub.status.idle":"2022-07-21T09:26:14.215163Z","shell.execute_reply.started":"2022-07-21T09:26:13.687943Z","shell.execute_reply":"2022-07-21T09:26:14.213992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that there is an order to type of Garage","metadata":{"papermill":{"duration":0.066812,"end_time":"2022-07-19T06:51:32.312434","exception":false,"start_time":"2022-07-19T06:51:32.245622","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We shall use ordinal encoder on these two features too","metadata":{"papermill":{"duration":0.066616,"end_time":"2022-07-19T06:51:32.446281","exception":false,"start_time":"2022-07-19T06:51:32.379665","status":"completed"},"tags":[]}},{"cell_type":"code","source":"t = []\nt.append(['Street', ['Grvl', 'Pave']])\nt.append(['Alley', ['No', 'Grvl', 'Pave']])\nt.append(['LotShape', [\"IR3\", 'IR2', 'IR1', 'Reg']])\nt.append(['BsmtExposure', ['No', 'Mn', 'Av', 'Gd']])\nt.append(['Functional', ['Sal', 'Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ']])\nt.append(['ExterCond', ['Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['ExterQual', ['Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['BsmtQual', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['BsmtCond', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['HeatingQC', ['Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['CentralAir', ['N', 'Y']])\nt.append(['KitchenQual', ['Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['FireplaceQu', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['GarageQual', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['GarageCond', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['PoolQC', ['No', 'Po', 'Fa', 'TA', 'Gd', 'Ex']])\nt.append(['Fence', ['No', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']])\nt.append(['PavedDrive', ['N', 'P', 'Y']])\nt.append(['LandSlope', ['Sev', 'Mod', 'Gtl']])\nt.append(['BsmtFinType1', ['No', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']])\nt.append(['BsmtFinType2', ['No', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']])\nt.append(['GarageFinish', ['No', 'Unf', 'RFn', 'Fin']])\n\nt.append(['Electrical', ['Mix', 'FuseP', 'FuseA', 'SBrkr']])\nt.append(['GarageType', ['No', 'CarPort', 'Detchd', '2Types', 'Basment', 'Attchd', 'BuiltIn']])","metadata":{"papermill":{"duration":0.086662,"end_time":"2022-07-19T06:51:32.600278","exception":false,"start_time":"2022-07-19T06:51:32.513616","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.217117Z","iopub.execute_input":"2022-07-21T09:26:14.217919Z","iopub.status.idle":"2022-07-21T09:26:14.236441Z","shell.execute_reply.started":"2022-07-21T09:26:14.217874Z","shell.execute_reply":"2022-07-21T09:26:14.235217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create a custom class, so that this can be integrated in the final pipeline ofthe model","metadata":{"papermill":{"duration":0.066042,"end_time":"2022-07-19T06:51:32.732295","exception":false,"start_time":"2022-07-19T06:51:32.666253","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def ordinal_encoder(X, train_data):\n    for i in range(len(t)):\n        #assigning column names and categories of each feature\n        col_name = t[i][0]\n        cat = t[i][1]\n        \n        #Instantiation of ordinal encoder\n        ode = OrdinalEncoder(categories=[cat], handle_unknown='use_encoded_value', unknown_value=-1, dtype=int)\n        \n        #encoding feature\n        ode.fit(train_data[col_name].values.reshape(-1, 1))\n        X[col_name] = ode.transform(X[col_name].values.reshape(-1, 1))\n        \n    return X","metadata":{"papermill":{"duration":0.079315,"end_time":"2022-07-19T06:51:32.878093","exception":false,"start_time":"2022-07-19T06:51:32.798778","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.237879Z","iopub.execute_input":"2022-07-21T09:26:14.238312Z","iopub.status.idle":"2022-07-21T09:26:14.253792Z","shell.execute_reply.started":"2022-07-21T09:26:14.238279Z","shell.execute_reply":"2022-07-21T09:26:14.252510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#encoding ordinal features by calling fit and transform on training data\ntrain = ordinal_encoder(train, train_data = train)","metadata":{"papermill":{"duration":0.104237,"end_time":"2022-07-19T06:51:33.048567","exception":false,"start_time":"2022-07-19T06:51:32.944330","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.255272Z","iopub.execute_input":"2022-07-21T09:26:14.255765Z","iopub.status.idle":"2022-07-21T09:26:14.296361Z","shell.execute_reply.started":"2022-07-21T09:26:14.255731Z","shell.execute_reply":"2022-07-21T09:26:14.295251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA & Feature Engineering**","metadata":{"papermill":{"duration":0.067194,"end_time":"2022-07-19T06:51:33.181718","exception":false,"start_time":"2022-07-19T06:51:33.114524","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We first look at the correlation matrix. This will help us find relations between features","metadata":{"papermill":{"duration":0.065906,"end_time":"2022-07-19T06:51:33.313896","exception":false,"start_time":"2022-07-19T06:51:33.247990","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#correlation matrix\ncorr_matrix = train.corr(method='pearson')\n#lower triangle of matrix\n#plt.figure(figsize=(13,10))\n#sns.heatmap(corr_matrix.where(np.tril((corr_matrix), k=-1).astype(bool)));","metadata":{"papermill":{"duration":0.091739,"end_time":"2022-07-19T06:51:33.471653","exception":false,"start_time":"2022-07-19T06:51:33.379914","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.306264Z","iopub.execute_input":"2022-07-21T09:26:14.306684Z","iopub.status.idle":"2022-07-21T09:26:14.330181Z","shell.execute_reply.started":"2022-07-21T09:26:14.306649Z","shell.execute_reply":"2022-07-21T09:26:14.329132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#picking the features with largest correlations with SalePrice\nx = corr_matrix.nlargest(10+1, 'SalePrice').index\ndf = train[list(x)]\n#correlation matrix of \ncorr_t10 = df.corr(method='pearson')\n#plotting matrix\nfig,ax = plt.subplots(figsize=(16, 8))\n#since the matrix is mirror image along main diagonal, we plot only lower traingle \nsns.heatmap(corr_t10.where(np.tril(corr_t10, k=-1).astype(bool)), \n            annot=True, fmt='.2f', annot_kws={'size': 12},cbar=False, ax=ax);","metadata":{"papermill":{"duration":0.536373,"end_time":"2022-07-19T06:51:34.073704","exception":false,"start_time":"2022-07-19T06:51:33.537331","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.331738Z","iopub.execute_input":"2022-07-21T09:26:14.332187Z","iopub.status.idle":"2022-07-21T09:26:14.827895Z","shell.execute_reply.started":"2022-07-21T09:26:14.332143Z","shell.execute_reply":"2022-07-21T09:26:14.826604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We shall also use MI score since pearson correlation only finds linear relatioship between variables","metadata":{"papermill":{"duration":0.069322,"end_time":"2022-07-19T06:51:34.211148","exception":false,"start_time":"2022-07-19T06:51:34.141826","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#numerical features selected\nnum_cols = train.select_dtypes(include=[int, float]).drop(['SalePrice'], axis=1).columns\n\n#MI score for numerical cols\nmi_score = mutual_info_regression(X=train[num_cols], y=train['SalePrice'])\ndf_mi_score = pd.Series(mi_score, index=num_cols, name='scores')\ndf_mi_score.sort_values(ascending=False, inplace=True)\n\n#plot\nfig,ax = plt.subplots(figsize=(13,6))\nsns.barplot(x=df_mi_score[:15], y=df_mi_score.index[:15], ax=ax)\nax.bar_label(ax.containers[0], fmt='%.2f');","metadata":{"papermill":{"duration":0.958661,"end_time":"2022-07-19T06:51:35.238702","exception":false,"start_time":"2022-07-19T06:51:34.280041","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:14.829362Z","iopub.execute_input":"2022-07-21T09:26:14.830644Z","iopub.status.idle":"2022-07-21T09:26:15.788639Z","shell.execute_reply.started":"2022-07-21T09:26:14.830600Z","shell.execute_reply":"2022-07-21T09:26:15.787288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that mi score for some of the features are ranked much higher, for example: YearBuilt","metadata":{"papermill":{"duration":0.069217,"end_time":"2022-07-19T06:51:35.377707","exception":false,"start_time":"2022-07-19T06:51:35.308490","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We select these few of these features and construct a pair plot scatter graph for more visualization","metadata":{"papermill":{"duration":0.069164,"end_time":"2022-07-19T06:51:35.516153","exception":false,"start_time":"2022-07-19T06:51:35.446989","status":"completed"},"tags":[]}},{"cell_type":"code","source":"cols = ['SalePrice', 'OverallQual', 'GrLivArea','GarageCars', 'GarageArea', 'TotalBsmtSF', 'ExterQual', 'YearBuilt']\nsns.pairplot(train[cols]);","metadata":{"papermill":{"duration":12.196982,"end_time":"2022-07-19T06:51:47.782296","exception":false,"start_time":"2022-07-19T06:51:35.585314","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:15.790389Z","iopub.execute_input":"2022-07-21T09:26:15.790748Z","iopub.status.idle":"2022-07-21T09:26:28.492139Z","shell.execute_reply.started":"2022-07-21T09:26:15.790717Z","shell.execute_reply":"2022-07-21T09:26:28.490898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Straight away, we can see some of the relationships between features\n- SalePrice seem to follow a steep linear relation or maybe be quadratic relation as well with GrLivArea, TotalBsmtSF, OverallQual as expected from correlation matrix\n- GarageArea and GarageCars also show a linear relation\n- As seen from mi score, YearBuilt also has a correlation with SalePrice, maybe exponential relation?\n","metadata":{"papermill":{"duration":0.07558,"end_time":"2022-07-19T06:51:47.934481","exception":false,"start_time":"2022-07-19T06:51:47.858901","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Lets look at some features one by one","metadata":{"papermill":{"duration":0.075971,"end_time":"2022-07-19T06:51:48.085786","exception":false,"start_time":"2022-07-19T06:51:48.009815","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- Utilities: Type of utilities available\n\t\t\n       AllPub\tAll public Utilities (E,G,W,& S)\t\n       NoSewr\tElectricity, Gas, and Water (Septic Tank)\n       NoSeWa\tElectricity and Gas Only\n       ELO\tElectricity only","metadata":{"papermill":{"duration":0.077476,"end_time":"2022-07-19T06:51:48.239396","exception":false,"start_time":"2022-07-19T06:51:48.161920","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#we check the number of unique values in utilities feature\nprint('in training data:\\n', train['Utilities'].value_counts())\nprint('\\nin test data:\\n', test['Utilities'].value_counts())\n\nfig,(ax1,ax2) = plt.subplots(1,2, figsize=(8,6))\n#training data\ntrain['Utilities'].value_counts().plot(kind='pie', autopct='%.2f', radius=1.5, ax=ax1)\nax1.set(title='Training data');\n\n#test data\ntest['Utilities'].value_counts().plot(kind='pie', autopct='%.2f', radius=1.5, ax=ax2)\nax2.set(title='Test data');\nplt.tight_layout()","metadata":{"papermill":{"duration":0.341902,"end_time":"2022-07-19T06:51:48.659540","exception":false,"start_time":"2022-07-19T06:51:48.317638","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.493869Z","iopub.execute_input":"2022-07-21T09:26:28.494227Z","iopub.status.idle":"2022-07-21T09:26:28.754175Z","shell.execute_reply.started":"2022-07-21T09:26:28.494196Z","shell.execute_reply":"2022-07-21T09:26:28.753347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What we see is that in both training and test data, we have one category of utility dominating the feature. Thus it would be wise to drop the feature to prevent overfitting due to other category of utility feature","metadata":{"papermill":{"duration":0.078662,"end_time":"2022-07-19T06:51:48.843043","exception":false,"start_time":"2022-07-19T06:51:48.764381","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping the utilities feature\ndrop_cols = []\ndrop_cols.append(['Utilities'])","metadata":{"papermill":{"duration":0.085058,"end_time":"2022-07-19T06:51:49.061501","exception":false,"start_time":"2022-07-19T06:51:48.976443","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.756025Z","iopub.execute_input":"2022-07-21T09:26:28.756828Z","iopub.status.idle":"2022-07-21T09:26:28.762390Z","shell.execute_reply.started":"2022-07-21T09:26:28.756782Z","shell.execute_reply":"2022-07-21T09:26:28.760816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Bathrooms in the house\n\n        BsmtFullBath: Basement full bathrooms\n        BsmtHalfBath: Basement half bathrooms\n        FullBath: Full bathrooms above grade\n        HalfBath: Half baths above grade","metadata":{"papermill":{"duration":0.076968,"end_time":"2022-07-19T06:51:49.216257","exception":false,"start_time":"2022-07-19T06:51:49.139289","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#bath features\nbath_cols = ['BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[bath_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[bath_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.088869,"end_time":"2022-07-19T06:51:49.382113","exception":false,"start_time":"2022-07-19T06:51:49.293244","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.763831Z","iopub.execute_input":"2022-07-21T09:26:28.765106Z","iopub.status.idle":"2022-07-21T09:26:28.786310Z","shell.execute_reply.started":"2022-07-21T09:26:28.765058Z","shell.execute_reply":"2022-07-21T09:26:28.785256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In both mi scores and pearson coefficient, we see that number of full baths has high effect on SalePrice, followed by half baths and basement bathrooms\n\nInstead of 4 features for bathrooms, we devise a metric (bathroom score), which gives an overall score to bathrooms in a house. We define the metric such way that higher weightage is given to FullBaths and HalfBath compared to basement baths (loosely based on mi score and correlation coefficient)","metadata":{"papermill":{"duration":0.075744,"end_time":"2022-07-19T06:51:49.534932","exception":false,"start_time":"2022-07-19T06:51:49.459188","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def bath_score(X):\n    #score = ['FullBath']*2 + ['HalfBath']*1 + ['BsmtFullBath']*0.5 + ['BsmtHalfBath']*0.5\n    X['bath_score'] = X.apply(lambda x: \n                              (2*x['FullBath'])+(1*x['HalfBath'])+(0.5*x['BsmtFullBath'])+(0.5*x['BsmtHalfBath']), axis=1)\n    \n    return X","metadata":{"papermill":{"duration":0.087373,"end_time":"2022-07-19T06:51:49.700207","exception":false,"start_time":"2022-07-19T06:51:49.612834","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.788321Z","iopub.execute_input":"2022-07-21T09:26:28.789051Z","iopub.status.idle":"2022-07-21T09:26:28.796111Z","shell.execute_reply.started":"2022-07-21T09:26:28.789006Z","shell.execute_reply":"2022-07-21T09:26:28.795224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = bath_score(train)","metadata":{"papermill":{"duration":0.135864,"end_time":"2022-07-19T06:51:49.913135","exception":false,"start_time":"2022-07-19T06:51:49.777271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.798494Z","iopub.execute_input":"2022-07-21T09:26:28.799964Z","iopub.status.idle":"2022-07-21T09:26:28.859009Z","shell.execute_reply.started":"2022-07-21T09:26:28.799915Z","shell.execute_reply":"2022-07-21T09:26:28.857823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,(ax1,ax2) = plt.subplots(1,2, figsize=(16, 6))\n#scatterplot\nsns.scatterplot(x=train.index, y='SalePrice', data=train, hue='bath_score', \n                palette='Spectral_r', ax=ax1)\n#boxplot\nsns.boxplot(x ='bath_score', y='SalePrice', data=train, ax=ax2);","metadata":{"papermill":{"duration":0.794735,"end_time":"2022-07-19T06:51:50.783202","exception":false,"start_time":"2022-07-19T06:51:49.988467","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:28.863122Z","iopub.execute_input":"2022-07-21T09:26:28.863447Z","iopub.status.idle":"2022-07-21T09:26:29.528313Z","shell.execute_reply.started":"2022-07-21T09:26:28.863419Z","shell.execute_reply":"2022-07-21T09:26:29.527492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We shall drop the 4 original bathroom features","metadata":{"papermill":{"duration":0.078896,"end_time":"2022-07-19T06:51:50.942526","exception":false,"start_time":"2022-07-19T06:51:50.863630","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping the bathroom feature\ndrop_cols.append(bath_cols)","metadata":{"papermill":{"duration":0.085937,"end_time":"2022-07-19T06:51:51.107931","exception":false,"start_time":"2022-07-19T06:51:51.021994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.529508Z","iopub.execute_input":"2022-07-21T09:26:29.530268Z","iopub.status.idle":"2022-07-21T09:26:29.534968Z","shell.execute_reply.started":"2022-07-21T09:26:29.530231Z","shell.execute_reply":"2022-07-21T09:26:29.533930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Porches","metadata":{"papermill":{"duration":0.079265,"end_time":"2022-07-19T06:51:51.267116","exception":false,"start_time":"2022-07-19T06:51:51.187851","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"        WoodDeckSF: Wood deck area in square feet\n        OpenPorchSF: Open porch area in square feet\n        EnclosedPorch: Enclosed porch area in square feet\n        3SsnPorch: Three season porch area in square feet\n        ScreenPorch: Screen porch area in square feet\n\n Instead of 5 different features, we combine these features to get total porch area of the house","metadata":{"papermill":{"duration":0.078314,"end_time":"2022-07-19T06:51:51.424317","exception":false,"start_time":"2022-07-19T06:51:51.346003","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#porch features\nporch_cols = ['WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[porch_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[porch_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.092618,"end_time":"2022-07-19T06:51:51.595562","exception":false,"start_time":"2022-07-19T06:51:51.502944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.536363Z","iopub.execute_input":"2022-07-21T09:26:29.536784Z","iopub.status.idle":"2022-07-21T09:26:29.554519Z","shell.execute_reply.started":"2022-07-21T09:26:29.536743Z","shell.execute_reply":"2022-07-21T09:26:29.553303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have a pretty similar mi score for all porches. We assign equal weightage to all types of porch","metadata":{"papermill":{"duration":0.078729,"end_time":"2022-07-19T06:51:51.753265","exception":false,"start_time":"2022-07-19T06:51:51.674536","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def porch_area(X):    \n    #summing porch areas of the house\n    X['porch_area'] = X[porch_cols].sum(axis=1)\n    return X","metadata":{"papermill":{"duration":0.089241,"end_time":"2022-07-19T06:51:51.921744","exception":false,"start_time":"2022-07-19T06:51:51.832503","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.556216Z","iopub.execute_input":"2022-07-21T09:26:29.556702Z","iopub.status.idle":"2022-07-21T09:26:29.562678Z","shell.execute_reply.started":"2022-07-21T09:26:29.556658Z","shell.execute_reply":"2022-07-21T09:26:29.561702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = porch_area(train)","metadata":{"papermill":{"duration":0.091308,"end_time":"2022-07-19T06:51:52.092192","exception":false,"start_time":"2022-07-19T06:51:52.000884","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.564100Z","iopub.execute_input":"2022-07-21T09:26:29.564398Z","iopub.status.idle":"2022-07-21T09:26:29.576533Z","shell.execute_reply.started":"2022-07-21T09:26:29.564371Z","shell.execute_reply":"2022-07-21T09:26:29.575387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(x='porch_area', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.295791,"end_time":"2022-07-19T06:51:52.466500","exception":false,"start_time":"2022-07-19T06:51:52.170709","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.578308Z","iopub.execute_input":"2022-07-21T09:26:29.578662Z","iopub.status.idle":"2022-07-21T09:26:29.751063Z","shell.execute_reply.started":"2022-07-21T09:26:29.578629Z","shell.execute_reply":"2022-07-21T09:26:29.749937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that many houses have 0 porch area, indicating no porches, thus we add another feature, denoting the number of porches in the house","metadata":{"papermill":{"duration":0.079491,"end_time":"2022-07-19T06:51:52.627082","exception":false,"start_time":"2022-07-19T06:51:52.547591","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def n_porch(X):    \n    #number of porches in the house\n    X['n_porch'] = X[porch_cols].gt(0).sum(axis=1)\n    return X","metadata":{"papermill":{"duration":0.089128,"end_time":"2022-07-19T06:51:52.796183","exception":false,"start_time":"2022-07-19T06:51:52.707055","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.752334Z","iopub.execute_input":"2022-07-21T09:26:29.752772Z","iopub.status.idle":"2022-07-21T09:26:29.758071Z","shell.execute_reply.started":"2022-07-21T09:26:29.752729Z","shell.execute_reply":"2022-07-21T09:26:29.757137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = n_porch(train)","metadata":{"papermill":{"duration":0.241046,"end_time":"2022-07-19T06:51:53.117369","exception":false,"start_time":"2022-07-19T06:51:52.876323","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.759011Z","iopub.execute_input":"2022-07-21T09:26:29.759335Z","iopub.status.idle":"2022-07-21T09:26:29.771710Z","shell.execute_reply.started":"2022-07-21T09:26:29.759307Z","shell.execute_reply":"2022-07-21T09:26:29.770655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,(ax1,ax2) = plt.subplots(1,2, figsize=(16,6))\nsns.scatterplot(x='porch_area', y='SalePrice', hue='n_porch', data=train, palette='Spectral',ax=ax1);\nsns.boxplot(x='n_porch', y='SalePrice', data=train, ax=ax2);","metadata":{"papermill":{"duration":0.584202,"end_time":"2022-07-19T06:51:53.781410","exception":false,"start_time":"2022-07-19T06:51:53.197208","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:29.772933Z","iopub.execute_input":"2022-07-21T09:26:29.773839Z","iopub.status.idle":"2022-07-21T09:26:30.278924Z","shell.execute_reply.started":"2022-07-21T09:26:29.773800Z","shell.execute_reply":"2022-07-21T09:26:30.277759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We drop the original porch features from our dataset","metadata":{"papermill":{"duration":0.081287,"end_time":"2022-07-19T06:51:53.945486","exception":false,"start_time":"2022-07-19T06:51:53.864199","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping the porch features\ndrop_cols.append(porch_cols)","metadata":{"papermill":{"duration":0.088883,"end_time":"2022-07-19T06:51:54.116400","exception":false,"start_time":"2022-07-19T06:51:54.027517","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.280372Z","iopub.execute_input":"2022-07-21T09:26:30.280740Z","iopub.status.idle":"2022-07-21T09:26:30.285706Z","shell.execute_reply.started":"2022-07-21T09:26:30.280707Z","shell.execute_reply":"2022-07-21T09:26:30.284204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Pool**","metadata":{"papermill":{"duration":0.083244,"end_time":"2022-07-19T06:51:54.280924","exception":false,"start_time":"2022-07-19T06:51:54.197680","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**PoolArea**: Pool area in square feet\n\n**PoolQC**: Pool quality\n\t\t\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       NA\tNo Pool","metadata":{"papermill":{"duration":0.082147,"end_time":"2022-07-19T06:51:54.445778","exception":false,"start_time":"2022-07-19T06:51:54.363631","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#distribution of pool area in training dataset\nprint('PoolArea in training data\\n',train['PoolArea'].value_counts())\n#distribution of pool area in test dataset\nprint('\\nPoolArea in test data\\n',test['PoolArea'].value_counts())","metadata":{"papermill":{"duration":0.093136,"end_time":"2022-07-19T06:51:54.621152","exception":false,"start_time":"2022-07-19T06:51:54.528016","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.287834Z","iopub.execute_input":"2022-07-21T09:26:30.288554Z","iopub.status.idle":"2022-07-21T09:26:30.304080Z","shell.execute_reply.started":"2022-07-21T09:26:30.288508Z","shell.execute_reply":"2022-07-21T09:26:30.302938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have two features for pool: PoolArea and PoolQC. We already have ordinal encoded PoolQC feature in earlier section","metadata":{"papermill":{"duration":0.081847,"end_time":"2022-07-19T06:51:54.784716","exception":false,"start_time":"2022-07-19T06:51:54.702869","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Since most of the houses do not have a pool, hence introducing a feature indicating presence of a pool(a categorical variable) would be more useful than PoolArea(a continuous variable)","metadata":{"papermill":{"duration":0.082735,"end_time":"2022-07-19T06:51:54.949450","exception":false,"start_time":"2022-07-19T06:51:54.866715","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def pool(X):    \n    #introducing new feature pool, indicating the presence of a pool in the house\n    #(falseValue, trueValue)[test == True]\n    X['pool'] = X['PoolArea'].apply(lambda x:(0, 1)[x > 0])\n    return X","metadata":{"papermill":{"duration":0.090525,"end_time":"2022-07-19T06:51:55.123294","exception":false,"start_time":"2022-07-19T06:51:55.032769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.305375Z","iopub.execute_input":"2022-07-21T09:26:30.306185Z","iopub.status.idle":"2022-07-21T09:26:30.313097Z","shell.execute_reply.started":"2022-07-21T09:26:30.306138Z","shell.execute_reply":"2022-07-21T09:26:30.312046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pool(train)","metadata":{"papermill":{"duration":0.094241,"end_time":"2022-07-19T06:51:55.299730","exception":false,"start_time":"2022-07-19T06:51:55.205489","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.314423Z","iopub.execute_input":"2022-07-21T09:26:30.315445Z","iopub.status.idle":"2022-07-21T09:26:30.326637Z","shell.execute_reply.started":"2022-07-21T09:26:30.315397Z","shell.execute_reply":"2022-07-21T09:26:30.325460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(y='SalePrice', x='pool', data=train, ci=None);","metadata":{"papermill":{"duration":0.24627,"end_time":"2022-07-19T06:51:55.629407","exception":false,"start_time":"2022-07-19T06:51:55.383137","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.328540Z","iopub.execute_input":"2022-07-21T09:26:30.329956Z","iopub.status.idle":"2022-07-21T09:26:30.493972Z","shell.execute_reply.started":"2022-07-21T09:26:30.329905Z","shell.execute_reply":"2022-07-21T09:26:30.492719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we have two features on pools now(PoolQC, pool), we combine them to generate a pool score of the house","metadata":{"papermill":{"duration":0.083876,"end_time":"2022-07-19T06:51:55.795517","exception":false,"start_time":"2022-07-19T06:51:55.711641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def pool_score(X):\n    X['pool_score'] = X.apply(lambda x: x['pool'] * x['PoolQC'], axis=1)\n    return X","metadata":{"papermill":{"duration":0.091737,"end_time":"2022-07-19T06:51:55.969354","exception":false,"start_time":"2022-07-19T06:51:55.877617","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.495592Z","iopub.execute_input":"2022-07-21T09:26:30.496043Z","iopub.status.idle":"2022-07-21T09:26:30.502356Z","shell.execute_reply.started":"2022-07-21T09:26:30.495999Z","shell.execute_reply":"2022-07-21T09:26:30.501250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pool_score(train)","metadata":{"papermill":{"duration":0.126255,"end_time":"2022-07-19T06:51:56.179604","exception":false,"start_time":"2022-07-19T06:51:56.053349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.503881Z","iopub.execute_input":"2022-07-21T09:26:30.504980Z","iopub.status.idle":"2022-07-21T09:26:30.549117Z","shell.execute_reply.started":"2022-07-21T09:26:30.504934Z","shell.execute_reply":"2022-07-21T09:26:30.547914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols.append(['PoolArea', 'PoolQC'])","metadata":{"papermill":{"duration":0.094384,"end_time":"2022-07-19T06:51:56.355952","exception":false,"start_time":"2022-07-19T06:51:56.261568","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.551650Z","iopub.execute_input":"2022-07-21T09:26:30.551958Z","iopub.status.idle":"2022-07-21T09:26:30.556679Z","shell.execute_reply.started":"2022-07-21T09:26:30.551930Z","shell.execute_reply":"2022-07-21T09:26:30.555812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Fireplaces**","metadata":{"papermill":{"duration":0.081033,"end_time":"2022-07-19T06:51:56.519786","exception":false,"start_time":"2022-07-19T06:51:56.438753","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Fireplaces: Number of fireplaces\n\nFireplaceQu: Fireplace quality\n\n       Ex\tExcellent - Exceptional Masonry Fireplace\n       Gd\tGood - Masonry Fireplace in main level\n       TA\tAverage - Prefabricated Fireplace in main living area or Masonry Fireplace in basement\n       Fa\tFair - Prefabricated Fireplace in basement\n       Po\tPoor - Ben Franklin Stove\n       NA\tNo Fireplace","metadata":{"papermill":{"duration":0.081171,"end_time":"2022-07-19T06:51:56.682237","exception":false,"start_time":"2022-07-19T06:51:56.601066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#distribution of fireplaces in training dataset\nprint('Fireplaces in training data\\n',train['Fireplaces'].value_counts())\n#distribution of fireplaces in test dataset\nprint('\\nFireplaces in test data\\n',test['Fireplaces'].value_counts())","metadata":{"papermill":{"duration":0.104255,"end_time":"2022-07-19T06:51:56.871203","exception":false,"start_time":"2022-07-19T06:51:56.766948","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.557674Z","iopub.execute_input":"2022-07-21T09:26:30.558138Z","iopub.status.idle":"2022-07-21T09:26:30.573359Z","shell.execute_reply.started":"2022-07-21T09:26:30.558109Z","shell.execute_reply":"2022-07-21T09:26:30.571852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We already have ordinal encoded FireplaceQu feature. We create a feature fireplace_score that combines both the fireplace features","metadata":{"papermill":{"duration":0.081836,"end_time":"2022-07-19T06:51:57.037844","exception":false,"start_time":"2022-07-19T06:51:56.956008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def fireplace_score(X):\n    X['fireplace_score'] = X['Fireplaces'] * X['FireplaceQu']\n    return X","metadata":{"papermill":{"duration":0.092838,"end_time":"2022-07-19T06:51:57.212543","exception":false,"start_time":"2022-07-19T06:51:57.119705","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.575098Z","iopub.execute_input":"2022-07-21T09:26:30.576615Z","iopub.status.idle":"2022-07-21T09:26:30.584786Z","shell.execute_reply.started":"2022-07-21T09:26:30.576546Z","shell.execute_reply":"2022-07-21T09:26:30.583496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#generating score\ntrain = fireplace_score(train)","metadata":{"papermill":{"duration":0.092292,"end_time":"2022-07-19T06:51:57.388305","exception":false,"start_time":"2022-07-19T06:51:57.296013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.586595Z","iopub.execute_input":"2022-07-21T09:26:30.587061Z","iopub.status.idle":"2022-07-21T09:26:30.596393Z","shell.execute_reply.started":"2022-07-21T09:26:30.587016Z","shell.execute_reply":"2022-07-21T09:26:30.595561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nsns.barplot(x='fireplace_score', y='SalePrice', data=train, ci=None);","metadata":{"papermill":{"duration":0.30149,"end_time":"2022-07-19T06:51:57.770664","exception":false,"start_time":"2022-07-19T06:51:57.469174","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:30.599250Z","iopub.execute_input":"2022-07-21T09:26:30.600371Z","iopub.status.idle":"2022-07-21T09:26:31.187097Z","shell.execute_reply.started":"2022-07-21T09:26:30.600321Z","shell.execute_reply":"2022-07-21T09:26:31.186057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping original features\ndrop_cols.append(['Fireplaces', 'FireplaceQu'])","metadata":{"papermill":{"duration":0.091412,"end_time":"2022-07-19T06:51:57.944779","exception":false,"start_time":"2022-07-19T06:51:57.853367","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.191873Z","iopub.execute_input":"2022-07-21T09:26:31.192659Z","iopub.status.idle":"2022-07-21T09:26:31.197287Z","shell.execute_reply.started":"2022-07-21T09:26:31.192612Z","shell.execute_reply":"2022-07-21T09:26:31.196533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Overall Rating**","metadata":{"papermill":{"duration":0.081018,"end_time":"2022-07-19T06:51:58.282531","exception":false,"start_time":"2022-07-19T06:51:58.201513","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"OverallQual: Rates the overall material and finish of the house\n\n       10\tVery Excellent\n       9\tExcellent\n       8\tVery Good\n       7\tGood\n       6\tAbove Average\n       5\tAverage\n       4\tBelow Average\n       3\tFair\n       2\tPoor\n       1\tVery Poor\n\t\nOverallCond: Rates the overall condition of the house\n\n       10\tVery Excellent\n       9\tExcellent\n       8\tVery Good\n       7\tGood\n       6\tAbove Average\t\n       5\tAverage\n       4\tBelow Average\t\n       3\tFair\n       2\tPoor\n       1\tVery Poor","metadata":{"papermill":{"duration":0.081214,"end_time":"2022-07-19T06:51:58.446756","exception":false,"start_time":"2022-07-19T06:51:58.365542","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We have features denoting overall quality and overall condition. We combine them to generate an overall score for the house","metadata":{"papermill":{"duration":0.081569,"end_time":"2022-07-19T06:51:58.609765","exception":false,"start_time":"2022-07-19T06:51:58.528196","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def overall_score(X):\n    X['overall_score'] = X['OverallQual'] * X['OverallCond']\n    return X","metadata":{"papermill":{"duration":0.089578,"end_time":"2022-07-19T06:51:58.781252","exception":false,"start_time":"2022-07-19T06:51:58.691674","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.198529Z","iopub.execute_input":"2022-07-21T09:26:31.199033Z","iopub.status.idle":"2022-07-21T09:26:31.208646Z","shell.execute_reply.started":"2022-07-21T09:26:31.199003Z","shell.execute_reply":"2022-07-21T09:26:31.207824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = overall_score(train)","metadata":{"papermill":{"duration":0.091918,"end_time":"2022-07-19T06:51:58.954697","exception":false,"start_time":"2022-07-19T06:51:58.862779","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.209749Z","iopub.execute_input":"2022-07-21T09:26:31.210713Z","iopub.status.idle":"2022-07-21T09:26:31.221468Z","shell.execute_reply.started":"2022-07-21T09:26:31.210680Z","shell.execute_reply":"2022-07-21T09:26:31.220644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,6))\nsns.barplot(x='overall_score', y='SalePrice', data=train, ci=None);","metadata":{"papermill":{"duration":0.440405,"end_time":"2022-07-19T06:51:59.478087","exception":false,"start_time":"2022-07-19T06:51:59.037682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.222851Z","iopub.execute_input":"2022-07-21T09:26:31.223416Z","iopub.status.idle":"2022-07-21T09:26:31.591984Z","shell.execute_reply.started":"2022-07-21T09:26:31.223380Z","shell.execute_reply":"2022-07-21T09:26:31.590819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bath features\noverall_cols = ['OverallQual', 'OverallCond']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[overall_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[overall_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.096952,"end_time":"2022-07-19T06:51:59.657940","exception":false,"start_time":"2022-07-19T06:51:59.560988","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.593665Z","iopub.execute_input":"2022-07-21T09:26:31.594003Z","iopub.status.idle":"2022-07-21T09:26:31.603813Z","shell.execute_reply.started":"2022-07-21T09:26:31.593971Z","shell.execute_reply":"2022-07-21T09:26:31.602779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping OverallCond feature\ndrop_cols.append(['OverallCond'])","metadata":{"papermill":{"duration":0.091111,"end_time":"2022-07-19T06:51:59.832229","exception":false,"start_time":"2022-07-19T06:51:59.741118","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.605287Z","iopub.execute_input":"2022-07-21T09:26:31.606364Z","iopub.status.idle":"2022-07-21T09:26:31.616002Z","shell.execute_reply.started":"2022-07-21T09:26:31.606326Z","shell.execute_reply":"2022-07-21T09:26:31.614867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Exterior**","metadata":{"papermill":{"duration":0.082978,"end_time":"2022-07-19T06:51:59.997600","exception":false,"start_time":"2022-07-19T06:51:59.914622","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"ExterQual: Evaluates the quality of the material on the exterior \n\t\t\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       Po\tPoor\n\t\t\nExterCond: Evaluates the present condition of the material on the exterior\n\t\t\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       Po\tPoor","metadata":{"papermill":{"duration":0.084118,"end_time":"2022-07-19T06:52:00.164751","exception":false,"start_time":"2022-07-19T06:52:00.080633","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We already have ordinal encoded both the features, we combine them to get an exterior score","metadata":{"papermill":{"duration":0.137457,"end_time":"2022-07-19T06:52:00.384192","exception":false,"start_time":"2022-07-19T06:52:00.246735","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def exter_score(X):\n    X['exter_score'] = X['ExterQual'] * X['ExterCond']\n    return X","metadata":{"papermill":{"duration":0.09136,"end_time":"2022-07-19T06:52:00.560034","exception":false,"start_time":"2022-07-19T06:52:00.468674","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.617271Z","iopub.execute_input":"2022-07-21T09:26:31.618176Z","iopub.status.idle":"2022-07-21T09:26:31.628113Z","shell.execute_reply.started":"2022-07-21T09:26:31.618141Z","shell.execute_reply":"2022-07-21T09:26:31.626725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = exter_score(train)","metadata":{"papermill":{"duration":0.098648,"end_time":"2022-07-19T06:52:00.743533","exception":false,"start_time":"2022-07-19T06:52:00.644885","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.630276Z","iopub.execute_input":"2022-07-21T09:26:31.631187Z","iopub.status.idle":"2022-07-21T09:26:31.643463Z","shell.execute_reply.started":"2022-07-21T09:26:31.631141Z","shell.execute_reply":"2022-07-21T09:26:31.642355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='exter_score', y='SalePrice', data=train, ci=None);","metadata":{"papermill":{"duration":0.300761,"end_time":"2022-07-19T06:52:01.130070","exception":false,"start_time":"2022-07-19T06:52:00.829309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.645612Z","iopub.execute_input":"2022-07-21T09:26:31.646355Z","iopub.status.idle":"2022-07-21T09:26:31.869033Z","shell.execute_reply.started":"2022-07-21T09:26:31.646308Z","shell.execute_reply":"2022-07-21T09:26:31.867896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bath features\nexter_cols = ['ExterCond', 'ExterQual']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[exter_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[exter_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.095759,"end_time":"2022-07-19T06:52:01.308265","exception":false,"start_time":"2022-07-19T06:52:01.212506","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.871047Z","iopub.execute_input":"2022-07-21T09:26:31.871741Z","iopub.status.idle":"2022-07-21T09:26:31.883026Z","shell.execute_reply.started":"2022-07-21T09:26:31.871694Z","shell.execute_reply":"2022-07-21T09:26:31.881799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping external condition feature\ndrop_cols.append(['ExterCond'])","metadata":{"papermill":{"duration":0.090598,"end_time":"2022-07-19T06:52:01.483524","exception":false,"start_time":"2022-07-19T06:52:01.392926","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.884607Z","iopub.execute_input":"2022-07-21T09:26:31.884995Z","iopub.status.idle":"2022-07-21T09:26:31.890553Z","shell.execute_reply.started":"2022-07-21T09:26:31.884956Z","shell.execute_reply":"2022-07-21T09:26:31.889186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Kitchen**","metadata":{"papermill":{"duration":0.082771,"end_time":"2022-07-19T06:52:01.649386","exception":false,"start_time":"2022-07-19T06:52:01.566615","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"KitchenAbvGr : Kitchens above grade\n\nKitchenQual: Kitchen quality\n\n       Ex\tExcellent\n       Gd\tGood\n       TA\tTypical/Average\n       Fa\tFair\n       Po\tPoor","metadata":{"papermill":{"duration":0.083119,"end_time":"2022-07-19T06:52:01.815065","exception":false,"start_time":"2022-07-19T06:52:01.731946","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We already have ordinal encoded KitchenQual, we combine them to get an kitchen score","metadata":{"papermill":{"duration":0.082996,"end_time":"2022-07-19T06:52:01.981725","exception":false,"start_time":"2022-07-19T06:52:01.898729","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def kitchen_score(X):\n    X['kitchen_score'] = X['KitchenAbvGr'] * X['KitchenQual']\n    return X","metadata":{"papermill":{"duration":0.091267,"end_time":"2022-07-19T06:52:02.156974","exception":false,"start_time":"2022-07-19T06:52:02.065707","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.891958Z","iopub.execute_input":"2022-07-21T09:26:31.892269Z","iopub.status.idle":"2022-07-21T09:26:31.900456Z","shell.execute_reply.started":"2022-07-21T09:26:31.892241Z","shell.execute_reply":"2022-07-21T09:26:31.899600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = kitchen_score(train)","metadata":{"papermill":{"duration":0.091412,"end_time":"2022-07-19T06:52:02.330891","exception":false,"start_time":"2022-07-19T06:52:02.239479","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.901740Z","iopub.execute_input":"2022-07-21T09:26:31.902216Z","iopub.status.idle":"2022-07-21T09:26:31.911606Z","shell.execute_reply.started":"2022-07-21T09:26:31.902186Z","shell.execute_reply":"2022-07-21T09:26:31.910633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='kitchen_score', y='SalePrice', data=train, ci=None);","metadata":{"papermill":{"duration":0.266288,"end_time":"2022-07-19T06:52:02.682143","exception":false,"start_time":"2022-07-19T06:52:02.415855","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:31.912693Z","iopub.execute_input":"2022-07-21T09:26:31.913716Z","iopub.status.idle":"2022-07-21T09:26:32.098339Z","shell.execute_reply.started":"2022-07-21T09:26:31.913682Z","shell.execute_reply":"2022-07-21T09:26:32.097208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bath features\nkitchen_cols = ['KitchenAbvGr', 'KitchenQual']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[kitchen_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[kitchen_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.097792,"end_time":"2022-07-19T06:52:02.863828","exception":false,"start_time":"2022-07-19T06:52:02.766036","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:32.099797Z","iopub.execute_input":"2022-07-21T09:26:32.100139Z","iopub.status.idle":"2022-07-21T09:26:32.110328Z","shell.execute_reply.started":"2022-07-21T09:26:32.100100Z","shell.execute_reply":"2022-07-21T09:26:32.109158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping KitchenAbvGr feature\ndrop_cols.append(['KitchenAbvGr'])","metadata":{"papermill":{"duration":0.281653,"end_time":"2022-07-19T06:52:03.228335","exception":false,"start_time":"2022-07-19T06:52:02.946682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:32.112391Z","iopub.execute_input":"2022-07-21T09:26:32.112858Z","iopub.status.idle":"2022-07-21T09:26:32.122386Z","shell.execute_reply.started":"2022-07-21T09:26:32.112815Z","shell.execute_reply":"2022-07-21T09:26:32.121181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **YearBuilt**: Original construction date\n","metadata":{"papermill":{"duration":0.083688,"end_time":"2022-07-19T06:52:03.395012","exception":false,"start_time":"2022-07-19T06:52:03.311324","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sns.scatterplot(x='YearBuilt', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.648083,"end_time":"2022-07-19T06:52:04.125326","exception":false,"start_time":"2022-07-19T06:52:03.477243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:32.124364Z","iopub.execute_input":"2022-07-21T09:26:32.124998Z","iopub.status.idle":"2022-07-21T09:26:32.342339Z","shell.execute_reply.started":"2022-07-21T09:26:32.124960Z","shell.execute_reply":"2022-07-21T09:26:32.341579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We use kmeans algorithm to divide the houses into clusters.\nTo decide upon the number of cluster to use, we use silhouette_score.","metadata":{"papermill":{"duration":0.083074,"end_time":"2022-07-19T06:52:04.292265","exception":false,"start_time":"2022-07-19T06:52:04.209191","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#we copy our data to check for optimal number of clusters\ntrial = train.copy()\n\nfeatures = ['YearBuilt']\n#features = ['last_remodel', 'YearBuilt']\nX_scaled = trial.loc[:, features]\nX_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n\ns = []\nfor n in range(2, 8):\n    kmeans = KMeans(n_clusters = n, random_state=0)\n    kmeans.fit(X_scaled)\n    s.append(silhouette_score(X_scaled, kmeans.labels_))\n    \nplt.plot(range(2, 8), s, '-');","metadata":{"papermill":{"duration":0.732683,"end_time":"2022-07-19T06:52:05.108173","exception":false,"start_time":"2022-07-19T06:52:04.375490","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:32.343814Z","iopub.execute_input":"2022-07-21T09:26:32.344109Z","iopub.status.idle":"2022-07-21T09:26:33.099844Z","shell.execute_reply.started":"2022-07-21T09:26:32.344082Z","shell.execute_reply":"2022-07-21T09:26:33.098739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that best silhouette score is obtained at n_clusters=3.\n\nSo we use n_clusters=3 as our hyperparameter in kmeans algorithm","metadata":{"papermill":{"duration":0.083327,"end_time":"2022-07-19T06:52:05.277539","exception":false,"start_time":"2022-07-19T06:52:05.194212","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def kmeans_yearblt(X, train_data):\n\n    #feature based on which clustering is done\n    features = ['YearBuilt']\n\n    #kmeans\n    kmeans = KMeans(n_clusters = 3, random_state=0)\n    kmeans.fit(train_data[features])\n    X['yr_group'] = kmeans.predict(X[features])\n    #convert into categorical type\n    #X['yr_group'] = X['yr_group'].astype('category')\n    return X","metadata":{"papermill":{"duration":0.096403,"end_time":"2022-07-19T06:52:05.459045","exception":false,"start_time":"2022-07-19T06:52:05.362642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:33.101310Z","iopub.execute_input":"2022-07-21T09:26:33.101663Z","iopub.status.idle":"2022-07-21T09:26:33.108996Z","shell.execute_reply.started":"2022-07-21T09:26:33.101633Z","shell.execute_reply":"2022-07-21T09:26:33.107676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = kmeans_yearblt(train, train)","metadata":{"papermill":{"duration":0.27773,"end_time":"2022-07-19T06:52:05.821545","exception":false,"start_time":"2022-07-19T06:52:05.543815","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:33.110707Z","iopub.execute_input":"2022-07-21T09:26:33.111056Z","iopub.status.idle":"2022-07-21T09:26:33.271409Z","shell.execute_reply.started":"2022-07-21T09:26:33.111025Z","shell.execute_reply":"2022-07-21T09:26:33.267976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting distribution\nsns.lmplot(x='YearBuilt', y='SalePrice', hue='yr_group', data=train, scatter_kws=dict(alpha=0.1));","metadata":{"papermill":{"duration":0.960152,"end_time":"2022-07-19T06:52:06.893375","exception":false,"start_time":"2022-07-19T06:52:05.933223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:33.274229Z","iopub.execute_input":"2022-07-21T09:26:33.278598Z","iopub.status.idle":"2022-07-21T09:26:34.208356Z","shell.execute_reply.started":"2022-07-21T09:26:33.278529Z","shell.execute_reply":"2022-07-21T09:26:34.207290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Basement**","metadata":{"papermill":{"duration":0.085752,"end_time":"2022-07-19T06:52:07.065392","exception":false,"start_time":"2022-07-19T06:52:06.979640","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"     BsmtQual: Evaluates the height of the basement\n\n       Ex\tExcellent (100+ inches)\t\n       Gd\tGood (90-99 inches)\n       TA\tTypical (80-89 inches)\n       Fa\tFair (70-79 inches)\n       Po\tPoor (<70 inches\n       NA\tNo Basement\n\t\t\nBsmtCond: Evaluates the general condition of the basement\n\n       Ex\tExcellent\n       Gd\tGood\n       TA\tTypical - slight dampness allowed\n       Fa\tFair - dampness or some cracking or settling\n       Po\tPoor - Severe cracking, settling, or wetness\n       NA\tNo Basement\n\t\nBsmtExposure: Refers to walkout or garden level walls\n\n       Gd\tGood Exposure\n       Av\tAverage Exposure (split levels or foyers typically score average or above)\t\n       Mn\tMimimum Exposure\n       No\tNo Exposure\n       NA\tNo Basement\n\t\nBsmtFinType1: Rating of basement finished area\n\n       GLQ\tGood Living Quarters\n       ALQ\tAverage Living Quarters\n       BLQ\tBelow Average Living Quarters\t\n       Rec\tAverage Rec Room\n       LwQ\tLow Quality\n       Unf\tUnfinshed\n       NA\tNo Basement\n\t\t\nBsmtFinSF1: Type 1 finished square feet\n\nBsmtFinType2: Rating of basement finished area (if multiple types)\n\n       GLQ\tGood Living Quarters\n       ALQ\tAverage Living Quarters\n       BLQ\tBelow Average Living Quarters\t\n       Rec\tAverage Rec Room\n       LwQ\tLow Quality\n       Unf\tUnfinshed\n       NA\tNo Basement\n\nBsmtFinSF2: Type 2 finished square feet\n\nBsmtUnfSF: Unfinished square feet of basement area\n\nTotalBsmtSF: Total square feet of basement area","metadata":{"papermill":{"duration":0.083811,"end_time":"2022-07-19T06:52:07.233964","exception":false,"start_time":"2022-07-19T06:52:07.150153","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We check distribution of basement area in our dataset","metadata":{"papermill":{"duration":0.085258,"end_time":"2022-07-19T06:52:07.403819","exception":false,"start_time":"2022-07-19T06:52:07.318561","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#basement area distribution in dataset\nfig,axes = plt.subplots(3,2, figsize=(16, 8))\naxes = axes.ravel()\n\n#1st basement area\nsns.histplot(x='BsmtFinSF1', data=train, kde=True, ax=axes[0])\naxes[0].set(title='1st Basement Area');\n\n#2nd basement area\nsns.histplot(x='BsmtFinSF2', data=train, kde=True, ax=axes[1])\naxes[1].set(title='2nd Basement Area');\n\n#unfinished basement area\nsns.histplot(x='BsmtUnfSF', data=train, kde=True, ax=axes[2])\naxes[2].set(title='Unfinished Basement Area');\n\n#unfinished basement area ratio\nsns.histplot(x=train['BsmtUnfSF']/train['TotalBsmtSF'], data=train, kde=True, ax=axes[3])\naxes[3].set(title='Ratio of Unfinished Basement Area');\n\n#total basement area ratio\nsns.histplot(x=train['TotalBsmtSF'], data=train, kde=True, ax=axes[4])\naxes[4].set(title='Total Basement Area');\n\nfig.delaxes(axes[5])\nplt.tight_layout()","metadata":{"papermill":{"duration":1.276502,"end_time":"2022-07-19T06:52:08.765414","exception":false,"start_time":"2022-07-19T06:52:07.488912","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:34.209508Z","iopub.execute_input":"2022-07-21T09:26:34.210195Z","iopub.status.idle":"2022-07-21T09:26:35.439186Z","shell.execute_reply.started":"2022-07-21T09:26:34.210163Z","shell.execute_reply":"2022-07-21T09:26:35.437901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data is highly skewed. We shall use log transformation on this feature\n\nAlso, most of the houses have higher unfnished basement area, thus we create basement score incorportating both finished and unfinshed basements","metadata":{"papermill":{"duration":0.109854,"end_time":"2022-07-19T06:52:08.963735","exception":false,"start_time":"2022-07-19T06:52:08.853881","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def basement_score(X):\n    \n    #logtransform\n    basement_cols = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF']\n    for col in basement_cols:\n        X[col] = np.log1p(X[col])\n\n    #houses with no basement\n    no_basement_idx = X[X['TotalBsmtSF'] == 0].index\n    #houses with basement\n    basement_idx = X[~(X['TotalBsmtSF'] == 0)].index\n    \n    #house with no basement is given 0 score\n    X.loc[no_basement_idx, 'basement_score'] = 0\n    \n    #houses with basement is alloted score based on area and rating\n    X.loc[basement_idx, 'basement_score'] = X.loc[basement_idx].apply(lambda x: \n                                                   ((x['BsmtFinType1']*x['BsmtFinSF1'])+\n                                                    (x['BsmtFinType2']*x['BsmtFinSF2'])+\n                                                    (x['BsmtUnfSF']))/x['TotalBsmtSF'], axis=1)\n    return X","metadata":{"papermill":{"duration":0.100515,"end_time":"2022-07-19T06:52:09.151393","exception":false,"start_time":"2022-07-19T06:52:09.050878","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.440831Z","iopub.execute_input":"2022-07-21T09:26:35.441213Z","iopub.status.idle":"2022-07-21T09:26:35.451904Z","shell.execute_reply.started":"2022-07-21T09:26:35.441181Z","shell.execute_reply":"2022-07-21T09:26:35.450658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = basement_score(train)","metadata":{"papermill":{"duration":0.16325,"end_time":"2022-07-19T06:52:09.399994","exception":false,"start_time":"2022-07-19T06:52:09.236744","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.453871Z","iopub.execute_input":"2022-07-21T09:26:35.454228Z","iopub.status.idle":"2022-07-21T09:26:35.534931Z","shell.execute_reply.started":"2022-07-21T09:26:35.454196Z","shell.execute_reply":"2022-07-21T09:26:35.533814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(x='basement_score', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.296372,"end_time":"2022-07-19T06:52:09.785164","exception":false,"start_time":"2022-07-19T06:52:09.488792","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.552188Z","iopub.execute_input":"2022-07-21T09:26:35.552617Z","iopub.status.idle":"2022-07-21T09:26:35.766443Z","shell.execute_reply.started":"2022-07-21T09:26:35.552560Z","shell.execute_reply":"2022-07-21T09:26:35.765307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#basement features\nbasement_cols = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtCond', 'BsmtQual']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[basement_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[basement_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.102413,"end_time":"2022-07-19T06:52:09.976770","exception":false,"start_time":"2022-07-19T06:52:09.874357","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.767920Z","iopub.execute_input":"2022-07-21T09:26:35.768911Z","iopub.status.idle":"2022-07-21T09:26:35.779835Z","shell.execute_reply.started":"2022-07-21T09:26:35.768873Z","shell.execute_reply":"2022-07-21T09:26:35.778639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping the original features\ndrop_cols.append(['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtCond'])","metadata":{"papermill":{"duration":0.096306,"end_time":"2022-07-19T06:52:10.161512","exception":false,"start_time":"2022-07-19T06:52:10.065206","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.781375Z","iopub.execute_input":"2022-07-21T09:26:35.782549Z","iopub.status.idle":"2022-07-21T09:26:35.788703Z","shell.execute_reply.started":"2022-07-21T09:26:35.782501Z","shell.execute_reply":"2022-07-21T09:26:35.787440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We also create some new features","metadata":{"papermill":{"duration":0.087771,"end_time":"2022-07-19T06:52:10.336672","exception":false,"start_time":"2022-07-19T06:52:10.248901","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"The price of a house depends upon how it compares to other houses in the same neighborhood","metadata":{"papermill":{"duration":0.087936,"end_time":"2022-07-19T06:52:10.514549","exception":false,"start_time":"2022-07-19T06:52:10.426613","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#creating a new feature indicating median of living area of other houses in the same neighborhood\ndef neigh_area(X, train_data):\n    #dictionary mapping each neighborhood to its median GrLivArea\n    median_area = train_data.groupby(['Neighborhood'])['GrLivArea'].agg('median').to_dict()\n    \n    #feature indicating whether the house is bigger than median living area of neighborhood\n    X['big_med_area'] = X.apply(lambda x: x['GrLivArea'] > median_area[x['Neighborhood']], axis=1)\n    X['big_med_area'] = X['big_med_area'].astype(int)\n    \n    return X","metadata":{"papermill":{"duration":0.098381,"end_time":"2022-07-19T06:52:10.878270","exception":false,"start_time":"2022-07-19T06:52:10.779889","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.791746Z","iopub.execute_input":"2022-07-21T09:26:35.792853Z","iopub.status.idle":"2022-07-21T09:26:35.801179Z","shell.execute_reply.started":"2022-07-21T09:26:35.792795Z","shell.execute_reply":"2022-07-21T09:26:35.799836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = neigh_area(train, train_data=train)","metadata":{"papermill":{"duration":0.134176,"end_time":"2022-07-19T06:52:11.099612","exception":false,"start_time":"2022-07-19T06:52:10.965436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.802966Z","iopub.execute_input":"2022-07-21T09:26:35.803435Z","iopub.status.idle":"2022-07-21T09:26:35.848855Z","shell.execute_reply.started":"2022-07-21T09:26:35.803389Z","shell.execute_reply":"2022-07-21T09:26:35.847644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nsns.barplot(x='big_med_area', y='SalePrice', data=train, ci=None)\nplt.xlabel('Bigger than median area of neighborhood');","metadata":{"papermill":{"duration":0.263589,"end_time":"2022-07-19T06:52:11.449593","exception":false,"start_time":"2022-07-19T06:52:11.186004","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:35.850693Z","iopub.execute_input":"2022-07-21T09:26:35.851853Z","iopub.status.idle":"2022-07-21T09:26:36.020807Z","shell.execute_reply.started":"2022-07-21T09:26:35.851815Z","shell.execute_reply":"2022-07-21T09:26:36.019721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def new_features(X):\n    #new feature measuring living area above grade out of total lot area \n    X['liv_ratio'] = X['GrLivArea']/X['LotArea']\n\n    #new feature measuring average room size\n    X['room_size'] = (X['1stFlrSF']+X['2ndFlrSF'])/X['TotRmsAbvGrd']\n\n    #new feature measuring low quality finished work\n    X['lowqual_ratio'] = X['LowQualFinSF']/(X['GrLivArea'])\n\n    #new featuer measuring total area of the house\n    X['allSF'] = X['GrLivArea'] + X['TotalBsmtSF']\n    return X","metadata":{"papermill":{"duration":0.096779,"end_time":"2022-07-19T06:52:11.634670","exception":false,"start_time":"2022-07-19T06:52:11.537891","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.022130Z","iopub.execute_input":"2022-07-21T09:26:36.022441Z","iopub.status.idle":"2022-07-21T09:26:36.028777Z","shell.execute_reply.started":"2022-07-21T09:26:36.022413Z","shell.execute_reply":"2022-07-21T09:26:36.027774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = new_features(train)","metadata":{"papermill":{"duration":0.099436,"end_time":"2022-07-19T06:52:11.820964","exception":false,"start_time":"2022-07-19T06:52:11.721528","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.030148Z","iopub.execute_input":"2022-07-21T09:26:36.030485Z","iopub.status.idle":"2022-07-21T09:26:36.044414Z","shell.execute_reply.started":"2022-07-21T09:26:36.030455Z","shell.execute_reply":"2022-07-21T09:26:36.043288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,axs = plt.subplots(2,2, figsize=(13,8))\naxs = axs.ravel()\n#liv_ratio vs SalePrice\nsns.scatterplot(x='liv_ratio', y='SalePrice', data=train, ax=axs[0]);\n#room size vs SalePrice\nsns.scatterplot(x='room_size', y='SalePrice', data=train, ax=axs[1]);\n#low quality finished work vs SalePrice\nsns.scatterplot(x='lowqual_ratio', y='SalePrice', data=train, ax=axs[2]);\n#total area\nsns.scatterplot(x='allSF', y='SalePrice', data=train, ax=axs[3]);\nplt.tight_layout()","metadata":{"papermill":{"duration":0.835598,"end_time":"2022-07-19T06:52:12.745926","exception":false,"start_time":"2022-07-19T06:52:11.910328","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.045689Z","iopub.execute_input":"2022-07-21T09:26:36.046446Z","iopub.status.idle":"2022-07-21T09:26:36.781682Z","shell.execute_reply.started":"2022-07-21T09:26:36.046406Z","shell.execute_reply":"2022-07-21T09:26:36.780532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#basement features\narea_cols = ['GrLivArea', 'LowQualFinSF', 'LotArea', 'TotRmsAbvGrd', '1stFlrSF', '2ndFlrSF']\n#mi scores\nprint('mi scores with SalePrice\\n', df_mi_score[area_cols].sort_values(ascending=False))\n#pearson correlation\nprint('\\npearson correlation coefficient\\n', corr_matrix.loc[area_cols, 'SalePrice'].sort_values(ascending=False))","metadata":{"papermill":{"duration":0.103336,"end_time":"2022-07-19T06:52:12.939096","exception":false,"start_time":"2022-07-19T06:52:12.835760","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.783060Z","iopub.execute_input":"2022-07-21T09:26:36.783482Z","iopub.status.idle":"2022-07-21T09:26:36.797920Z","shell.execute_reply.started":"2022-07-21T09:26:36.783449Z","shell.execute_reply":"2022-07-21T09:26:36.796750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols.append(['LowQualFinSF', 'lowqual_ratio', 'LotArea'])","metadata":{"papermill":{"duration":0.099803,"end_time":"2022-07-19T06:52:13.128676","exception":false,"start_time":"2022-07-19T06:52:13.028873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.799385Z","iopub.execute_input":"2022-07-21T09:26:36.800165Z","iopub.status.idle":"2022-07-21T09:26:36.806121Z","shell.execute_reply.started":"2022-07-21T09:26:36.800128Z","shell.execute_reply":"2022-07-21T09:26:36.805062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Garage**\n\t\t\nGarageCars: Size of garage in car capacity\n\nGarageArea: Size of garage in square feet","metadata":{"papermill":{"duration":0.089953,"end_time":"2022-07-19T06:52:13.308284","exception":false,"start_time":"2022-07-19T06:52:13.218331","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corr_matrix['GarageArea']['GarageCars']","metadata":{"papermill":{"duration":0.102763,"end_time":"2022-07-19T06:52:13.500312","exception":false,"start_time":"2022-07-19T06:52:13.397549","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.807421Z","iopub.execute_input":"2022-07-21T09:26:36.808491Z","iopub.status.idle":"2022-07-21T09:26:36.819317Z","shell.execute_reply.started":"2022-07-21T09:26:36.808455Z","shell.execute_reply":"2022-07-21T09:26:36.818510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that GarageArea has a high linear relation with GarageCars, which also makes sense logically","metadata":{"papermill":{"duration":0.090147,"end_time":"2022-07-19T06:52:13.679653","exception":false,"start_time":"2022-07-19T06:52:13.589506","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#plotting distribution of garage area \nsns.scatterplot(x='GarageArea', y='SalePrice', data=train, hue='GarageCars', palette='Spectral');","metadata":{"papermill":{"duration":0.483803,"end_time":"2022-07-19T06:52:14.253138","exception":false,"start_time":"2022-07-19T06:52:13.769335","status":"completed"},"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:36.820454Z","iopub.execute_input":"2022-07-21T09:26:36.821181Z","iopub.status.idle":"2022-07-21T09:26:37.199681Z","shell.execute_reply.started":"2022-07-21T09:26:36.821145Z","shell.execute_reply":"2022-07-21T09:26:37.198454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Garage Cars has well defined clusters. \nThus we choose to drop GarageArea and keep GarageCars in our feature set","metadata":{"papermill":{"duration":0.089727,"end_time":"2022-07-19T06:52:14.434954","exception":false,"start_time":"2022-07-19T06:52:14.345227","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_cols.append(['GarageArea'])","metadata":{"papermill":{"duration":0.0993,"end_time":"2022-07-19T06:52:14.625523","exception":false,"start_time":"2022-07-19T06:52:14.526223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.201158Z","iopub.execute_input":"2022-07-21T09:26:37.201497Z","iopub.status.idle":"2022-07-21T09:26:37.206719Z","shell.execute_reply.started":"2022-07-21T09:26:37.201462Z","shell.execute_reply":"2022-07-21T09:26:37.205485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **MiscFeature**","metadata":{"papermill":{"duration":0.090603,"end_time":"2022-07-19T06:52:14.805965","exception":false,"start_time":"2022-07-19T06:52:14.715362","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"MiscFeature: Miscellaneous feature not covered in other categories\n\t\t\n       Elev\tElevator\n       Gar2\t2nd Garage (if not described in garage section)\n       Othr\tOther\n       Shed\tShed (over 100 SF)\n       TenC\tTennis Court\n       NA\tNone\n\t\t\nMiscVal: $Value of miscellaneous feature","metadata":{"papermill":{"duration":0.091294,"end_time":"2022-07-19T06:52:14.989113","exception":false,"start_time":"2022-07-19T06:52:14.897819","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We have additional features in house, along with their values.\nWe don't need to know what kind of feature it is when we have its value.\nSo keep only the MiscVal feature and drop the MiscFeature column","metadata":{"papermill":{"duration":0.092702,"end_time":"2022-07-19T06:52:15.174478","exception":false,"start_time":"2022-07-19T06:52:15.081776","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping the feature\ndrop_cols.append(['MiscFeature'])","metadata":{"papermill":{"duration":0.102613,"end_time":"2022-07-19T06:52:15.368866","exception":false,"start_time":"2022-07-19T06:52:15.266253","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.208486Z","iopub.execute_input":"2022-07-21T09:26:37.208827Z","iopub.status.idle":"2022-07-21T09:26:37.218850Z","shell.execute_reply.started":"2022-07-21T09:26:37.208798Z","shell.execute_reply":"2022-07-21T09:26:37.217685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Year Remodelled**\n\n        YearRemodAdd: Remodel date (same as construction date if no remodeling or additions)\n        YrSold: Year Sold (YYYY)","metadata":{"papermill":{"duration":0.092424,"end_time":"2022-07-19T06:52:15.611659","exception":false,"start_time":"2022-07-19T06:52:15.519235","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We create a feature indicating number of years since last remodelling of the house","metadata":{"papermill":{"duration":0.089757,"end_time":"2022-07-19T06:52:15.792763","exception":false,"start_time":"2022-07-19T06:52:15.703006","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def yrs_remodel(X):\n    X['yrs_remodel'] = X['YrSold'] - X['YearRemodAdd']\n    return X","metadata":{"papermill":{"duration":0.101511,"end_time":"2022-07-19T06:52:15.984299","exception":false,"start_time":"2022-07-19T06:52:15.882788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.221194Z","iopub.execute_input":"2022-07-21T09:26:37.222019Z","iopub.status.idle":"2022-07-21T09:26:37.231394Z","shell.execute_reply.started":"2022-07-21T09:26:37.221977Z","shell.execute_reply":"2022-07-21T09:26:37.230596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = yrs_remodel(train)","metadata":{"papermill":{"duration":0.09886,"end_time":"2022-07-19T06:52:16.174110","exception":false,"start_time":"2022-07-19T06:52:16.075250","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.232719Z","iopub.execute_input":"2022-07-21T09:26:37.233960Z","iopub.status.idle":"2022-07-21T09:26:37.244315Z","shell.execute_reply.started":"2022-07-21T09:26:37.233921Z","shell.execute_reply":"2022-07-21T09:26:37.243051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot\nfig,(ax1,ax2) = plt.subplots(1,2, figsize=(16,8))\n#scatterplot\nsns.scatterplot(x='yrs_remodel', y='SalePrice', data=train, ax=ax1)\nax1.set_xlabel('Years since last remodel')\n#barplot\nsns.barplot(x='yrs_remodel', y='SalePrice', data=train, ci=None, ax=ax2)\nax2.set_xlabel('Years since last remodel')\nax2.set_xticks([]);\n#ax2.tick_params(axis='x', rotation=90);","metadata":{"papermill":{"duration":0.606714,"end_time":"2022-07-19T06:52:16.872033","exception":false,"start_time":"2022-07-19T06:52:16.265319","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.245510Z","iopub.execute_input":"2022-07-21T09:26:37.246216Z","iopub.status.idle":"2022-07-21T09:26:37.779892Z","shell.execute_reply.started":"2022-07-21T09:26:37.246181Z","shell.execute_reply":"2022-07-21T09:26:37.778999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that higher the number of years since last remodelling of the house, lower is the price\n\nWe divide the last_remodel feature into clusters using KMeans","metadata":{"papermill":{"duration":0.09242,"end_time":"2022-07-19T06:52:17.057414","exception":false,"start_time":"2022-07-19T06:52:16.964994","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#we copy our data to check for optimal number of clusters\ntrial = train.copy()\n\nfeatures = ['yrs_remodel', 'YearBuilt']\nX_scaled = trial.loc[:, features]\nX_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n\ns = []\nfor n in range(2, 8):\n    kmeans = KMeans(n_clusters = n, random_state=0)\n    kmeans.fit(X_scaled)\n    s.append(silhouette_score(X_scaled, kmeans.labels_))\n    \nplt.plot(range(2, 8), s, '-');","metadata":{"papermill":{"duration":2.305104,"end_time":"2022-07-19T06:52:19.454470","exception":false,"start_time":"2022-07-19T06:52:17.149366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:37.780955Z","iopub.execute_input":"2022-07-21T09:26:37.781751Z","iopub.status.idle":"2022-07-21T09:26:39.051586Z","shell.execute_reply.started":"2022-07-21T09:26:37.781716Z","shell.execute_reply":"2022-07-21T09:26:39.049452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that best silhouette score is obtained at n_clusters=4\n\nSo we use n_clusters=4 as our hyperparameter in kmeans algorithm","metadata":{"papermill":{"duration":0.093333,"end_time":"2022-07-19T06:52:19.643994","exception":false,"start_time":"2022-07-19T06:52:19.550661","status":"completed"},"tags":[]}},{"cell_type":"raw","source":"class kmeans_lastremodel():\n    def __init__(self):\n        self.features = ['yrs_remodel', 'YearBuilt']\n        self.kmeans_fitted = []\n    def fit_predict(self, X):\n        #scaling features\n        X_scaled = X[self.features]\n        X_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n        \n        #fitting KMeans\n        kmeans = KMeans(n_clusters = 4, random_state=0)\n        self.kmeans_fitted.append(kmeans.fit(X_scaled))\n        \n        #predicting cluster\n        X['remodel_group'] = kmeans.predict(X_scaled)\n        #convert into categorical type\n        X['remodel_group'] = X['remodel_group'].astype('category')\n        return X\n    \n    def predict(self, X):\n        for k in self.kmeans_fitted:\n            X['remodel_group'] = k.predict(X[self.features])\n        return X","metadata":{"papermill":{"duration":0.098262,"end_time":"2022-07-19T06:52:20.031904","exception":false,"start_time":"2022-07-19T06:52:19.933642","status":"completed"},"tags":[]}},{"cell_type":"raw","source":"last_remodel = kmeans_lastremodel()","metadata":{"papermill":{"duration":0.095416,"end_time":"2022-07-19T06:52:20.226903","exception":false,"start_time":"2022-07-19T06:52:20.131487","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def last_remodel(X, train_data):\n\n    #feature based on which clustering is done\n    train_data['yrs_remodel'] = train_data['YrSold'] - train_data['YearRemodAdd']\n    features = ['yrs_remodel', 'YearBuilt']\n\n    #scaling training data feature\n    train_scaled = train_data.loc[:, features]\n    train_scaled = (train_scaled - train_scaled.mean(axis=0)) / train_scaled.std(axis=0)\n\n    #kmeans\n    kmeans = KMeans(n_clusters = 4, random_state=0)\n    kmeans.fit(train_scaled)\n    \n    #scaling\n    X_scaled = X.loc[:, features]\n    X_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n    \n    X['remodel_group'] = kmeans.predict(X_scaled[features])\n    #convert into categorical type\n    #X['remodel_group'] = X['remodel_group'].astype('category')\n    \n    return X\n#dropping the YearBuilt feature\n#train.drop(['YearBuilt'], axis=1, inplace=True)","metadata":{"papermill":{"duration":0.110403,"end_time":"2022-07-19T06:52:20.431897","exception":false,"start_time":"2022-07-19T06:52:20.321494","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:39.053005Z","iopub.execute_input":"2022-07-21T09:26:39.053892Z","iopub.status.idle":"2022-07-21T09:26:39.063468Z","shell.execute_reply.started":"2022-07-21T09:26:39.053856Z","shell.execute_reply":"2022-07-21T09:26:39.062661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clustering\ntrain = last_remodel(train, train)","metadata":{"papermill":{"duration":0.189924,"end_time":"2022-07-19T06:52:20.715339","exception":false,"start_time":"2022-07-19T06:52:20.525415","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:39.064846Z","iopub.execute_input":"2022-07-21T09:26:39.065148Z","iopub.status.idle":"2022-07-21T09:26:39.138902Z","shell.execute_reply.started":"2022-07-21T09:26:39.065120Z","shell.execute_reply":"2022-07-21T09:26:39.137735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting distribution\nsns.scatterplot(x='yrs_remodel', y='SalePrice', hue='remodel_group', data=train, palette='Spectral');\nplt.xlabel('Years since last remodelling');","metadata":{"papermill":{"duration":0.454734,"end_time":"2022-07-19T06:52:21.273962","exception":false,"start_time":"2022-07-19T06:52:20.819228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:39.140377Z","iopub.execute_input":"2022-07-21T09:26:39.140827Z","iopub.status.idle":"2022-07-21T09:26:39.782659Z","shell.execute_reply.started":"2022-07-21T09:26:39.140782Z","shell.execute_reply":"2022-07-21T09:26:39.781505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping original features\ndrop_cols.append(['YearBuilt', 'YrSold', 'yrs_remodel'])","metadata":{"papermill":{"duration":0.104815,"end_time":"2022-07-19T06:52:21.479222","exception":false,"start_time":"2022-07-19T06:52:21.374407","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:39.783976Z","iopub.execute_input":"2022-07-21T09:26:39.784276Z","iopub.status.idle":"2022-07-21T09:26:39.789232Z","shell.execute_reply.started":"2022-07-21T09:26:39.784249Z","shell.execute_reply":"2022-07-21T09:26:39.788138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Heating**\n\nHeating: Type of heating\n\t\t\n       Floor\tFloor Furnace\n       GasA\tGas forced warm air furnace\n       GasW\tGas hot water or steam heat\n       Grav\tGravity furnace\t\n       OthW\tHot water or steam heat other than gas\n       Wall\tWall furnace\n\t\t\nHeatingQC: Heating quality and condition\n\n       Ex\tExcellent\n       Gd\tGood\n       TA\tAverage/Typical\n       Fa\tFair\n       Po\tPoor","metadata":{"papermill":{"duration":0.100426,"end_time":"2022-07-19T06:52:21.681953","exception":false,"start_time":"2022-07-19T06:52:21.581527","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We have already encoded HeatingQC, we look at Heating feature","metadata":{"papermill":{"duration":0.09718,"end_time":"2022-07-19T06:52:21.873356","exception":false,"start_time":"2022-07-19T06:52:21.776176","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig,(ax1,ax2) = plt.subplots(1,2,figsize=(13,6))\n#training data\ntrain['Heating'].value_counts().plot(kind='pie', autopct='%.2f', ax=ax1);\n\n#test data\ntest['Heating'].value_counts().plot(kind='pie', autopct='%.2f', ax=ax2);","metadata":{"papermill":{"duration":0.355366,"end_time":"2022-07-19T06:52:22.322540","exception":false,"start_time":"2022-07-19T06:52:21.967174","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:39.790532Z","iopub.execute_input":"2022-07-21T09:26:39.790889Z","iopub.status.idle":"2022-07-21T09:26:40.062123Z","shell.execute_reply.started":"2022-07-21T09:26:39.790860Z","shell.execute_reply":"2022-07-21T09:26:40.060626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that Heating feature is dominated by a single category in both test and training data.\nWe choose to drop the feature","metadata":{"papermill":{"duration":0.093211,"end_time":"2022-07-19T06:52:22.520710","exception":false,"start_time":"2022-07-19T06:52:22.427499","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping feature\ndrop_cols.append(['Heating'])","metadata":{"papermill":{"duration":0.100813,"end_time":"2022-07-19T06:52:22.714163","exception":false,"start_time":"2022-07-19T06:52:22.613350","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.064144Z","iopub.execute_input":"2022-07-21T09:26:40.065189Z","iopub.status.idle":"2022-07-21T09:26:40.070531Z","shell.execute_reply.started":"2022-07-21T09:26:40.065137Z","shell.execute_reply":"2022-07-21T09:26:40.069401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Veneer**\n\nMasVnrType: Masonry veneer type\n\n       BrkCmn\tBrick Common\n       BrkFace\tBrick Face\n       CBlock\tCinder Block\n       None\tNone\n       Stone\tStone\n\t\nMasVnrArea: Masonry veneer area in square feet","metadata":{"papermill":{"duration":0.093906,"end_time":"2022-07-19T06:52:22.900410","exception":false,"start_time":"2022-07-19T06:52:22.806504","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sns.scatterplot(x='MasVnrArea', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.318239,"end_time":"2022-07-19T06:52:23.311399","exception":false,"start_time":"2022-07-19T06:52:22.993160","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.072075Z","iopub.execute_input":"2022-07-21T09:26:40.073193Z","iopub.status.idle":"2022-07-21T09:26:40.297408Z","shell.execute_reply.started":"2022-07-21T09:26:40.073144Z","shell.execute_reply":"2022-07-21T09:26:40.296631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that most of the houses do not have veneer. We create a feature indicating whether the house has veneer","metadata":{"papermill":{"duration":0.091421,"end_time":"2022-07-19T06:52:23.495358","exception":false,"start_time":"2022-07-19T06:52:23.403937","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def veneer(X):\n    X['veneer'] = X['MasVnrArea'].apply(lambda x: (0, 1)[x>0])\n    return X","metadata":{"papermill":{"duration":0.10302,"end_time":"2022-07-19T06:52:23.692621","exception":false,"start_time":"2022-07-19T06:52:23.589601","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.298626Z","iopub.execute_input":"2022-07-21T09:26:40.298952Z","iopub.status.idle":"2022-07-21T09:26:40.304562Z","shell.execute_reply.started":"2022-07-21T09:26:40.298922Z","shell.execute_reply":"2022-07-21T09:26:40.303418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = veneer(train)","metadata":{"papermill":{"duration":0.107934,"end_time":"2022-07-19T06:52:23.893761","exception":false,"start_time":"2022-07-19T06:52:23.785827","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.305938Z","iopub.execute_input":"2022-07-21T09:26:40.306313Z","iopub.status.idle":"2022-07-21T09:26:40.317695Z","shell.execute_reply.started":"2022-07-21T09:26:40.306251Z","shell.execute_reply":"2022-07-21T09:26:40.316767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='veneer', y='SalePrice', data=train);","metadata":{"papermill":{"duration":0.537592,"end_time":"2022-07-19T06:52:24.524978","exception":false,"start_time":"2022-07-19T06:52:23.987386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.319440Z","iopub.execute_input":"2022-07-21T09:26:40.320170Z","iopub.status.idle":"2022-07-21T09:26:40.501836Z","shell.execute_reply.started":"2022-07-21T09:26:40.320125Z","shell.execute_reply":"2022-07-21T09:26:40.500652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax1 = plt.subplots(1,figsize=(6,6))\nsns.scatterplot(x='ExterQual', hue='MasVnrType', y='SalePrice', data=train, ax=ax1);","metadata":{"papermill":{"duration":0.651359,"end_time":"2022-07-19T06:52:25.269569","exception":false,"start_time":"2022-07-19T06:52:24.618210","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.503603Z","iopub.execute_input":"2022-07-21T09:26:40.504034Z","iopub.status.idle":"2022-07-21T09:26:40.852833Z","shell.execute_reply.started":"2022-07-21T09:26:40.503992Z","shell.execute_reply":"2022-07-21T09:26:40.851746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that type of veneer is well captured by ExterQual feature. So we choose to drop the MasVnrType feature","metadata":{"papermill":{"duration":0.092752,"end_time":"2022-07-19T06:52:25.456575","exception":false,"start_time":"2022-07-19T06:52:25.363823","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#dropping MasVnrType feature\ndrop_cols.append(['MasVnrType'])","metadata":{"papermill":{"duration":0.102582,"end_time":"2022-07-19T06:52:25.653867","exception":false,"start_time":"2022-07-19T06:52:25.551285","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.854736Z","iopub.execute_input":"2022-07-21T09:26:40.855671Z","iopub.status.idle":"2022-07-21T09:26:40.861488Z","shell.execute_reply.started":"2022-07-21T09:26:40.855625Z","shell.execute_reply":"2022-07-21T09:26:40.860229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **MSSubClass**\n\nMSSubClass: Identifies the type of dwelling involved in the sale.\t\n\n        20\t1-STORY 1946 & NEWER ALL STYLES\n        30\t1-STORY 1945 & OLDER\n        40\t1-STORY W/FINISHED ATTIC ALL AGES\n        45\t1-1/2 STORY - UNFINISHED ALL AGES\n        50\t1-1/2 STORY FINISHED ALL AGES\n        60\t2-STORY 1946 & NEWER\n        70\t2-STORY 1945 & OLDER\n        75\t2-1/2 STORY ALL AGES\n        80\tSPLIT OR MULTI-LEVEL\n        85\tSPLIT FOYER\n        90\tDUPLEX - ALL STYLES AND AGES\n       120\t1-STORY PUD (Planned Unit Development) - 1946 & NEWER\n       150\t1-1/2 STORY PUD - ALL AGES\n       160\t2-STORY PUD - 1946 & NEWER\n       180\tPUD - MULTILEVEL - INCL SPLIT LEV/FOYER\n       190\t2 FAMILY CONVERSION - ALL STYLES AND AGES","metadata":{"papermill":{"duration":0.09471,"end_time":"2022-07-19T06:52:25.844397","exception":false,"start_time":"2022-07-19T06:52:25.749687","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We see that MSSubClass comprises of two other features\n\nBldgType: Type of dwelling\n\t\t\n       1Fam\tSingle-family Detached\t\n       2FmCon\tTwo-family Conversion; originally built as one-family dwelling\n       Duplx\tDuplex\n       TwnhsE\tTownhouse End Unit\n       TwnhsI\tTownhouse Inside Unit\n\t\nHouseStyle: Style of dwelling\n\t\n       1Story\tOne story\n       1.5Fin\tOne and one-half story: 2nd level finished\n       1.5Unf\tOne and one-half story: 2nd level unfinished\n       2Story\tTwo story\n       2.5Fin\tTwo and one-half story: 2nd level finished\n       2.5Unf\tTwo and one-half story: 2nd level unfinished\n       SFoyer\tSplit Foyer\n       SLvl\tSplit Level","metadata":{"papermill":{"duration":0.095509,"end_time":"2022-07-19T06:52:26.035114","exception":false,"start_time":"2022-07-19T06:52:25.939605","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We choose to drop other two features","metadata":{"papermill":{"duration":0.09422,"end_time":"2022-07-19T06:52:26.225793","exception":false,"start_time":"2022-07-19T06:52:26.131573","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_cols.append(['BldgType', 'HouseStyle'])","metadata":{"papermill":{"duration":0.103398,"end_time":"2022-07-19T06:52:26.423244","exception":false,"start_time":"2022-07-19T06:52:26.319846","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.862819Z","iopub.execute_input":"2022-07-21T09:26:40.863147Z","iopub.status.idle":"2022-07-21T09:26:40.871107Z","shell.execute_reply.started":"2022-07-21T09:26:40.863118Z","shell.execute_reply":"2022-07-21T09:26:40.870332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We convert the MSSubClass into categorical type","metadata":{"papermill":{"duration":0.096229,"end_time":"2022-07-19T06:52:26.616525","exception":false,"start_time":"2022-07-19T06:52:26.520296","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def MSSubClass(X):\n    X = X.replace({'MSSubClass' : {20 : 'SC20', 30 : 'SC30', 40 : 'SC40', 45 : 'SC45', \n                                       50 : 'SC50', 60 : 'SC60', 70 : 'SC70', 75 : 'SC75', \n                                       80 : 'SC80', 85 : 'SC85', 90 : 'SC90', 120 : 'SC120', \n                                       150 : 'SC150', 160 : 'SC160', 180 : 'SC180', 190 : 'SC190'}})\n    return X","metadata":{"papermill":{"duration":0.106185,"end_time":"2022-07-19T06:52:26.819078","exception":false,"start_time":"2022-07-19T06:52:26.712893","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.872531Z","iopub.execute_input":"2022-07-21T09:26:40.873199Z","iopub.status.idle":"2022-07-21T09:26:40.883637Z","shell.execute_reply.started":"2022-07-21T09:26:40.873159Z","shell.execute_reply":"2022-07-21T09:26:40.882812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = MSSubClass(train)","metadata":{"papermill":{"duration":0.10758,"end_time":"2022-07-19T06:52:27.022680","exception":false,"start_time":"2022-07-19T06:52:26.915100","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.885053Z","iopub.execute_input":"2022-07-21T09:26:40.885681Z","iopub.status.idle":"2022-07-21T09:26:40.899415Z","shell.execute_reply.started":"2022-07-21T09:26:40.885650Z","shell.execute_reply":"2022-07-21T09:26:40.898631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Neighborhood**\n\nNeighborhood: Physical locations within Ames city limits\n\n       Blmngtn\tBloomington Heights\n       Blueste\tBluestem\n       BrDale\tBriardale\n       BrkSide\tBrookside\n       ClearCr\tClear Creek\n       CollgCr\tCollege Creek\n       Crawfor\tCrawford\n       Edwards\tEdwards\n       Gilbert\tGilbert\n       IDOTRR\tIowa DOT and Rail Road\n       MeadowV\tMeadow Village\n       Mitchel\tMitchell\n       Names\tNorth Ames\n       NoRidge\tNorthridge\n       NPkVill\tNorthpark Villa\n       NridgHt\tNorthridge Heights\n       NWAmes\tNorthwest Ames\n       OldTown\tOld Town\n       SWISU\tSouth & West of Iowa State University\n       Sawyer\tSawyer\n       SawyerW\tSawyer West\n       Somerst\tSomerset\n       StoneBr\tStone Brook\n       Timber\tTimberland\n       Veenker\tVeenker","metadata":{"papermill":{"duration":0.093932,"end_time":"2022-07-19T06:52:27.211275","exception":false,"start_time":"2022-07-19T06:52:27.117343","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We know that price of a house is determined by its locality","metadata":{"papermill":{"duration":0.093521,"end_time":"2022-07-19T06:52:27.398839","exception":false,"start_time":"2022-07-19T06:52:27.305318","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train['Neighborhood'].value_counts()","metadata":{"papermill":{"duration":0.108573,"end_time":"2022-07-19T06:52:27.601367","exception":false,"start_time":"2022-07-19T06:52:27.492794","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.900765Z","iopub.execute_input":"2022-07-21T09:26:40.901090Z","iopub.status.idle":"2022-07-21T09:26:40.914379Z","shell.execute_reply.started":"2022-07-21T09:26:40.901061Z","shell.execute_reply":"2022-07-21T09:26:40.913250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observe high cardinality","metadata":{"papermill":{"duration":0.09666,"end_time":"2022-07-19T06:52:27.792789","exception":false,"start_time":"2022-07-19T06:52:27.696129","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We use target encoding for Neighborhood feature","metadata":{"papermill":{"duration":0.093894,"end_time":"2022-07-19T06:52:27.982188","exception":false,"start_time":"2022-07-19T06:52:27.888294","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CrossFoldEncoder:\n    def __init__(self, encoder, **kwargs):\n        self.encoder_ = encoder\n        self.kwargs_ = kwargs  # keyword arguments for the encoder\n        self.cv_ = KFold(n_splits=5) #cross validation splits\n\n    # Fit an encoder on one split and transform the feature on the\n    # other. Iterating over the splits in all folds gives a complete\n    # transformation. We also now have one trained encoder on each\n    # fold.\n    def fit_transform(self, X, cols):\n        self.fitted_encoders_ = []\n        self.cols_ = cols\n        X_encoded = []\n        y=X.pop('SalePrice')\n        for idx_train, idx_encode in self.cv_.split(X):\n            #instantiate encoder\n            fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)\n            #fitting encoder on 4 parts of training CV\n            fitted_encoder.fit(\n                X.iloc[idx_train, :], y.iloc[idx_train],\n            )\n            #transforming on 1 part after fitting\n            X_encoded.append(fitted_encoder.transform(X.iloc[idx_encode, :])[cols])\n            #storing fitted encoders\n            self.fitted_encoders_.append(fitted_encoder)\n        #creating dataframe by concatenating 5 results\n        X_encoded = pd.concat(X_encoded)\n        #renaming the column\n        X_encoded.columns = [name + \"_encoded\" for name in X_encoded.columns]\n        X = X.join(X_encoded)\n        X = X.join(y.to_frame(name='SalePrice'))\n        return X\n\n    # To transform the test data, average the encodings learned from\n    # each fold.\n    def transform(self, X):\n        X_encoded_list = []\n        for fitted_encoder in self.fitted_encoders_:\n            #tranforming\n            X_encoded = fitted_encoder.transform(X)\n            #appending target encoded features\n            X_encoded_list.append(X_encoded[self.cols_])\n        #taking average\n        X_encoded = reduce(\n            lambda x, y: x.add(y, fill_value=0), X_encoded_list\n        ) / len(X_encoded_list)\n        X_encoded.columns = [name + \"_encoded\" for name in X_encoded.columns]\n        X = X.join(X_encoded)\n        return X","metadata":{"papermill":{"duration":0.111698,"end_time":"2022-07-19T06:52:28.378459","exception":false,"start_time":"2022-07-19T06:52:28.266761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.915964Z","iopub.execute_input":"2022-07-21T09:26:40.916533Z","iopub.status.idle":"2022-07-21T09:26:40.931227Z","shell.execute_reply.started":"2022-07-21T09:26:40.916501Z","shell.execute_reply":"2022-07-21T09:26:40.930130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def target_encoder(X, training_data):\n    encoder = CrossFoldEncoder(MEstimateEncoder, m=2)\n    if ('SalePrice' in X.columns):\n        X = encoder.fit_transform(X, cols=['Neighborhood'])\n    else:\n        _ = encoder.fit_transform(training_data, cols=['Neighborhood'])\n        X = encoder.transform(X)\n    \n    return X","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:40.932829Z","iopub.execute_input":"2022-07-21T09:26:40.933722Z","iopub.status.idle":"2022-07-21T09:26:40.947965Z","shell.execute_reply.started":"2022-07-21T09:26:40.933689Z","shell.execute_reply":"2022-07-21T09:26:40.946783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = target_encoder(train, train)","metadata":{"papermill":{"duration":0.105306,"end_time":"2022-07-19T06:52:28.578936","exception":false,"start_time":"2022-07-19T06:52:28.473630","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:40.949332Z","iopub.execute_input":"2022-07-21T09:26:40.949840Z","iopub.status.idle":"2022-07-21T09:26:41.069437Z","shell.execute_reply.started":"2022-07-21T09:26:40.949809Z","shell.execute_reply":"2022-07-21T09:26:41.068166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols.append(['Neighborhood'])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.070907Z","iopub.execute_input":"2022-07-21T09:26:41.071241Z","iopub.status.idle":"2022-07-21T09:26:41.076697Z","shell.execute_reply.started":"2022-07-21T09:26:41.071208Z","shell.execute_reply":"2022-07-21T09:26:41.075259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **LandSlope**\n\nWe have two features describing slope of property\n    \n    LandSlope: Slope of property\n\t\t\n       Gtl\tGentle slope\n       Mod\tModerate Slope\t\n       Sev\tSevere Slope\n       \n    LandContour: Flatness of the property\n\n       Lvl\tNear Flat/Level\t\n       Bnk\tBanked - Quick and significant rise from street grade to building\n       HLS\tHillside - Significant slope from side to side\n       Low\tDepression","metadata":{"papermill":{"duration":0.098264,"end_time":"2022-07-19T06:52:28.772770","exception":false,"start_time":"2022-07-19T06:52:28.674506","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We have already encoded LandSlope, we drop the LandContour feature","metadata":{"papermill":{"duration":0.095125,"end_time":"2022-07-19T06:52:28.965188","exception":false,"start_time":"2022-07-19T06:52:28.870063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#train.drop(['LandContour'], axis=1, inplace=True)\ndrop_cols.append(['LandContour'])","metadata":{"papermill":{"duration":0.10445,"end_time":"2022-07-19T06:52:29.165789","exception":false,"start_time":"2022-07-19T06:52:29.061339","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.077903Z","iopub.execute_input":"2022-07-21T09:26:41.078191Z","iopub.status.idle":"2022-07-21T09:26:41.088403Z","shell.execute_reply.started":"2022-07-21T09:26:41.078163Z","shell.execute_reply":"2022-07-21T09:26:41.087439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We label encode the remaining features","metadata":{"papermill":{"duration":0.094465,"end_time":"2022-07-19T06:52:29.355014","exception":false,"start_time":"2022-07-19T06:52:29.260549","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trial = train.copy()","metadata":{"papermill":{"duration":0.0941,"end_time":"2022-07-19T06:52:29.543478","exception":false,"start_time":"2022-07-19T06:52:29.449378","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.089456Z","iopub.execute_input":"2022-07-21T09:26:41.089767Z","iopub.status.idle":"2022-07-21T09:26:41.102812Z","shell.execute_reply.started":"2022-07-21T09:26:41.089738Z","shell.execute_reply":"2022-07-21T09:26:41.101759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.select_dtypes(exclude='object').columns","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.104481Z","iopub.execute_input":"2022-07-21T09:26:41.104944Z","iopub.status.idle":"2022-07-21T09:26:41.119899Z","shell.execute_reply.started":"2022-07-21T09:26:41.104899Z","shell.execute_reply":"2022-07-21T09:26:41.118654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Skewness**","metadata":{}},{"cell_type":"code","source":"num_cols = train.select_dtypes(exclude='object').columns.tolist()\n\n# Check the skew of all numerical features\n\nskewed_feats = train[num_cols].apply(lambda x: skew(x)).sort_values(ascending=False)\nskewness = pd.DataFrame({'Skew' :skewed_feats})\nskewness.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.123004Z","iopub.execute_input":"2022-07-21T09:26:41.123348Z","iopub.status.idle":"2022-07-21T09:26:41.159517Z","shell.execute_reply.started":"2022-07-21T09:26:41.123319Z","shell.execute_reply":"2022-07-21T09:26:41.158390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transform features with skewness more than 0.75","metadata":{}},{"cell_type":"code","source":"def transform(X):\n    #numerical features\n    num_cols = X.select_dtypes(exclude='object').columns.tolist()\n    #target variable\n    if ('SalePrice' in num_cols):\n        X['SalePrice'] = np.log1p(X['SalePrice'])\n        num_cols.remove('SalePrice')\n    \n    #finding skewness\n    skewed_feats = X[num_cols].apply(lambda x: skew(x)).sort_values(ascending=False)\n    skewness = pd.DataFrame({'Skew' :skewed_feats})\n    \n    skewness = skewness[abs(skewness) > 0.75]\n    cols = skewness.index.tolist()\n    \n    from scipy.special import boxcox1p\n    lam = 0.15\n    for col in cols:\n    #all_data[feat] += 1\n        X[col] = boxcox1p(X[col], lam)\n    return X\n    #transforming features","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.160801Z","iopub.execute_input":"2022-07-21T09:26:41.161219Z","iopub.status.idle":"2022-07-21T09:26:41.170337Z","shell.execute_reply.started":"2022-07-21T09:26:41.161177Z","shell.execute_reply":"2022-07-21T09:26:41.169286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = transform(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.172141Z","iopub.execute_input":"2022-07-21T09:26:41.172438Z","iopub.status.idle":"2022-07-21T09:26:41.243639Z","shell.execute_reply.started":"2022-07-21T09:26:41.172410Z","shell.execute_reply":"2022-07-21T09:26:41.242648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the distribution of 'SalePrice'\nfig, (ax1,ax2) = plt.subplots(1,2, figsize=(15,6))\n\nsns.histplot(x='SalePrice', data=train, kde=True, ax=ax1)\nax1.set(ylabel='frequency')\nax1.set(xlabel='SalePrice')\nax1.set(title='SalePrice Distribution');\n\n#QQplot\nstats.probplot(train['SalePrice'], plot=ax2)\nax2.set_title(\"QQplot of SalePrice\");","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.245006Z","iopub.execute_input":"2022-07-21T09:26:41.245369Z","iopub.status.idle":"2022-07-21T09:26:41.665644Z","shell.execute_reply.started":"2022-07-21T09:26:41.245336Z","shell.execute_reply":"2022-07-21T09:26:41.664586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is a much better distribution!!","metadata":{}},{"cell_type":"code","source":"def label_encode(X):\n    cols = ['MSSubClass', 'MSZoning', 'LotConfig', 'Condition1', 'Condition2', 'RoofStyle', 'RoofMatl',\n                 'Exterior1st', 'Exterior2nd', 'Foundation', 'SaleType', 'SaleCondition']\n    \n    X[cols] = X[cols].astype('category')\n    for col in cols:\n        X[col] = X[col].cat.codes\n    return X","metadata":{"papermill":{"duration":0.396345,"end_time":"2022-07-19T06:52:30.036129","exception":false,"start_time":"2022-07-19T06:52:29.639784","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.667212Z","iopub.execute_input":"2022-07-21T09:26:41.667655Z","iopub.status.idle":"2022-07-21T09:26:41.673744Z","shell.execute_reply.started":"2022-07-21T09:26:41.667618Z","shell.execute_reply":"2022-07-21T09:26:41.672697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot_encode(X):\n    cols = ['MSSubClass', 'MSZoning', 'LotConfig', 'Condition1', 'Condition2', 'RoofStyle', 'RoofMatl',\n                 'Exterior1st', 'Exterior2nd', 'Foundation', 'SaleType', 'SaleCondition']\n    \n    X = pd.get_dummies(X, columns=cols)\n    return X","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:41.675078Z","iopub.execute_input":"2022-07-21T09:26:41.675477Z","iopub.status.idle":"2022-07-21T09:26:41.685749Z","shell.execute_reply.started":"2022-07-21T09:26:41.675445Z","shell.execute_reply":"2022-07-21T09:26:41.684760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = onehot_encode(train)","metadata":{"papermill":{"duration":0.126928,"end_time":"2022-07-19T06:52:30.259229","exception":false,"start_time":"2022-07-19T06:52:30.132301","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.687104Z","iopub.execute_input":"2022-07-21T09:26:41.687412Z","iopub.status.idle":"2022-07-21T09:26:41.715500Z","shell.execute_reply.started":"2022-07-21T09:26:41.687385Z","shell.execute_reply":"2022-07-21T09:26:41.714525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **Dropping Columns**","metadata":{"papermill":{"duration":0.095547,"end_time":"2022-07-19T06:52:32.494390","exception":false,"start_time":"2022-07-19T06:52:32.398843","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We had made a list of columns to be dropped throughout our notebook.\nWe finally drop those columns before modelling","metadata":{"papermill":{"duration":0.099114,"end_time":"2022-07-19T06:52:32.688727","exception":false,"start_time":"2022-07-19T06:52:32.589613","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_cols","metadata":{"papermill":{"duration":0.109071,"end_time":"2022-07-19T06:52:32.899038","exception":false,"start_time":"2022-07-19T06:52:32.789967","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.716790Z","iopub.execute_input":"2022-07-21T09:26:41.717089Z","iopub.status.idle":"2022-07-21T09:26:41.726300Z","shell.execute_reply.started":"2022-07-21T09:26:41.717062Z","shell.execute_reply":"2022-07-21T09:26:41.724870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping features\ndef drop_features(X):\n    drop_list= [col for col_list in drop_cols for col in col_list]\n    X.drop(drop_list, inplace=True, axis=1)\n    return X","metadata":{"papermill":{"duration":0.107927,"end_time":"2022-07-19T06:52:33.103087","exception":false,"start_time":"2022-07-19T06:52:32.995160","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.728537Z","iopub.execute_input":"2022-07-21T09:26:41.729585Z","iopub.status.idle":"2022-07-21T09:26:41.740197Z","shell.execute_reply.started":"2022-07-21T09:26:41.729523Z","shell.execute_reply":"2022-07-21T09:26:41.739160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = drop_features(train)","metadata":{"papermill":{"duration":0.107311,"end_time":"2022-07-19T06:52:33.306373","exception":false,"start_time":"2022-07-19T06:52:33.199062","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.741641Z","iopub.execute_input":"2022-07-21T09:26:41.742427Z","iopub.status.idle":"2022-07-21T09:26:41.753278Z","shell.execute_reply.started":"2022-07-21T09:26:41.742390Z","shell.execute_reply":"2022-07-21T09:26:41.752373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We shall create a pipeline that combines all our feature engineering and transformations that we made on training data","metadata":{"papermill":{"duration":0.0954,"end_time":"2022-07-19T06:52:33.497112","exception":false,"start_time":"2022-07-19T06:52:33.401712","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def pipeline(X, training_data):\n    \n    # imputing missing values\n    X = missing_imputer(X, training_data)\n    \n    # ordinal encoding\n    X = ordinal_encoder(X, training_data)\n    \n    # target encoding\n    X = target_encoder(X, training_data)\n    \n    #  bathroom score\n    X = bath_score(X)\n    \n    #  porch area\n    X = porch_area(X)\n    \n    #  number of porches\n    X = n_porch(X)\n    \n    #  presence of pool\n    X = pool(X)\n    \n    #  pool score\n    X = pool_score(X)\n    \n    #  Fireplace score\n    X = fireplace_score(X)\n    \n    #  Overall score\n    X = overall_score(X)\n    \n    #  External score\n    X = exter_score(X)\n    \n    #  Kitchen score\n    X = kitchen_score(X)\n    \n    #  Clustering YearBuilt feature\n    X = kmeans_yearblt(X, training_data)\n    \n    #  Basement score\n    X = basement_score(X)\n    \n    #  Median Neighborhood Area\n    X = neigh_area(X, training_data)\n    \n    #  New features based on area\n    X = new_features(X)\n    \n    #  Years since last remodelling\n    X = yrs_remodel(X)\n    \n    # Clustering based on years since last remodel\n    X = last_remodel(X, training_data)\n    \n    # Veneer\n    X = veneer(X)\n    \n    # MSSubClass\n    X = MSSubClass(X)\n    \n    # target encoding\n    #X = target_encoder(X, training_data)\n    \n    # Skewness\n    X = transform(X)\n    \n    # label encode\n    X = label_encode(X)\n    \n    # Dropping features\n    X = drop_features(X)\n    \n    return X","metadata":{"papermill":{"duration":0.111651,"end_time":"2022-07-19T06:52:33.908528","exception":false,"start_time":"2022-07-19T06:52:33.796877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.754527Z","iopub.execute_input":"2022-07-21T09:26:41.755373Z","iopub.status.idle":"2022-07-21T09:26:41.765990Z","shell.execute_reply.started":"2022-07-21T09:26:41.755337Z","shell.execute_reply":"2022-07-21T09:26:41.764837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_copy = data.copy()\ndf_train = pipeline(data_copy, training_data = data_copy)\n\n#prepared training dataset\nX_train = df_train.copy()\ny_train = X_train.pop('SalePrice')","metadata":{"papermill":{"duration":0.486962,"end_time":"2022-07-19T06:52:34.492078","exception":false,"start_time":"2022-07-19T06:52:34.005116","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:41.767052Z","iopub.execute_input":"2022-07-21T09:26:41.767550Z","iopub.status.idle":"2022-07-21T09:26:42.335341Z","shell.execute_reply.started":"2022-07-21T09:26:41.767517Z","shell.execute_reply":"2022-07-21T09:26:42.333996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_copy = data.copy()\ntest_copy = test.copy()\ndf_test = pipeline(test_copy, data_copy)\n\n#prepared test dataset\nX_test = df_test.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:42.337080Z","iopub.execute_input":"2022-07-21T09:26:42.337530Z","iopub.status.idle":"2022-07-21T09:26:42.928540Z","shell.execute_reply.started":"2022-07-21T09:26:42.337485Z","shell.execute_reply":"2022-07-21T09:26:42.927538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=0.95)\nX_pca = pca.fit_transform(X_train)\nX_test = pca.transform(X_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = pd.DataFrame(X_pca, columns = list(range(X_pca.shape[1])))\nX_test = pd.DataFrame(X_test, columns = list(range(X_test.shape[1])))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Modelling** ","metadata":{}},{"cell_type":"markdown","source":"Cross-Validation Strategy","metadata":{}},{"cell_type":"code","source":"def rmsle(model):\n    kf = KFold(5, shuffle=True, random_state=7)\n    score = cross_val_score(model, X_train, y_train, scoring= 'neg_mean_squared_error', cv=kf)\n    rmsle = np.sqrt(-score)\n    return rmsle.mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:42.929875Z","iopub.execute_input":"2022-07-21T09:26:42.930186Z","iopub.status.idle":"2022-07-21T09:26:42.938820Z","shell.execute_reply.started":"2022-07-21T09:26:42.930158Z","shell.execute_reply":"2022-07-21T09:26:42.937501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBoost model","metadata":{}},{"cell_type":"code","source":"xgb = XGBRegressor()","metadata":{"papermill":{"duration":12.545094,"end_time":"2022-07-19T06:52:47.937604","exception":false,"start_time":"2022-07-19T06:52:35.392510","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:26:42.940471Z","iopub.execute_input":"2022-07-21T09:26:42.940852Z","iopub.status.idle":"2022-07-21T09:26:42.952468Z","shell.execute_reply.started":"2022-07-21T09:26:42.940821Z","shell.execute_reply":"2022-07-21T09:26:42.951275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Light GBM","metadata":{}},{"cell_type":"code","source":"lgb = LGBMRegressor()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:42.954392Z","iopub.execute_input":"2022-07-21T09:26:42.954858Z","iopub.status.idle":"2022-07-21T09:26:42.963409Z","shell.execute_reply.started":"2022-07-21T09:26:42.954814Z","shell.execute_reply":"2022-07-21T09:26:42.962553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Linear Regression model with L1 regularization","metadata":{}},{"cell_type":"code","source":"lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=11))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:42.964558Z","iopub.execute_input":"2022-07-21T09:26:42.965546Z","iopub.status.idle":"2022-07-21T09:26:42.975887Z","shell.execute_reply.started":"2022-07-21T09:26:42.965510Z","shell.execute_reply":"2022-07-21T09:26:42.975037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We check error on individual models","metadata":{}},{"cell_type":"code","source":"print('for lasso model:',rmsle(lasso))\nprint('for xgb model:',rmsle(xgb))\nprint('for lgb model:', rmsle(lgb))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:42.977086Z","iopub.execute_input":"2022-07-21T09:26:42.977740Z","iopub.status.idle":"2022-07-21T09:26:47.469050Z","shell.execute_reply.started":"2022-07-21T09:26:42.977704Z","shell.execute_reply":"2022-07-21T09:26:47.468034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Combining Predictions by averaging models","metadata":{}},{"cell_type":"code","source":"class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, models):\n        self.models = models\n        \n    # we define clones of the original models to fit the data in\n    def fit(self, X, y):\n        self.models_ = [clone(x) for x in self.models]\n        # Train cloned base models\n        for model in self.models_:\n            model.fit(X, y)\n        return self\n    \n    #Now we do the predictions for cloned models and average them\n    def predict(self, X):\n        predictions = np.column_stack([\n            model.predict(X) for model in self.models_\n        ])\n        return np.mean(predictions, axis=1)   ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:47.473188Z","iopub.execute_input":"2022-07-21T09:26:47.473828Z","iopub.status.idle":"2022-07-21T09:26:47.485330Z","shell.execute_reply.started":"2022-07-21T09:26:47.473788Z","shell.execute_reply":"2022-07-21T09:26:47.484428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_models = AveragingModels(models = (lgb, xgb))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:47.487025Z","iopub.execute_input":"2022-07-21T09:26:47.488836Z","iopub.status.idle":"2022-07-21T09:26:47.499164Z","shell.execute_reply.started":"2022-07-21T09:26:47.488774Z","shell.execute_reply":"2022-07-21T09:26:47.497975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We check error on combined models","metadata":{}},{"cell_type":"code","source":"rmsle(averaged_models)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:47.500874Z","iopub.execute_input":"2022-07-21T09:26:47.501618Z","iopub.status.idle":"2022-07-21T09:26:51.490869Z","shell.execute_reply.started":"2022-07-21T09:26:47.501555Z","shell.execute_reply":"2022-07-21T09:26:51.489969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have reduction in error, we ensemble the model by use of stacking","metadata":{}},{"cell_type":"code","source":"class stacking(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, level0_models, level1_model, n_folds=3):\n        self.level0_models = level0_models\n        self.level1_model = level1_model\n        self.n_folds = n_folds\n        \n    #training level-0 models\n    def fit(self, X, y):\n        self.fit_models = []\n        #predictions from level0 models\n        kf = KFold(self.n_folds, shuffle=True, random_state=2)\n        \n        for i, model in enumerate(self.level0_models):\n            #list of n models(ith level0 model) trained on n-1 folds\n            self.fit_models.append([])\n            model_pred = np.zeros((X.shape[0], len(self.level0_models)))\n            \n            #cross folds\n            for train_idx, val_idx in kf.split(X):\n                #fitting on training set\n                model.fit(X.iloc[train_idx], y.iloc[train_idx])\n                #print(X[train_idx])\n                #predictions on validation set\n                model_pred[val_idx, i] = model.predict(X.iloc[val_idx])\n                #print(model_pred)\n                #storing the fitted models\n                self.fit_models[i].append(model)\n        \n        #training level 1 model\n        self.level1_model.fit(model_pred, y)\n        return self\n    \n    def predict(self, X):\n        y_pred = np.zeros((X.shape[0], len(self.fit_models)))\n        for j, fit_model in enumerate(self.fit_models):\n            model_pred = np.zeros((X.shape[0], self.n_folds))\n            #prediction of n models(jth level0 model)\n            for i, model in enumerate(fit_model):\n                model_pred[:, i] = model.predict(X)\n            #mean of predictions of n models\n            y_pred[:, j] = model_pred.mean(axis=1)\n        #predictions from level1 model\n        return self.level1_model.predict(y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:51.492249Z","iopub.execute_input":"2022-07-21T09:26:51.492800Z","iopub.status.idle":"2022-07-21T09:26:51.505996Z","shell.execute_reply.started":"2022-07-21T09:26:51.492767Z","shell.execute_reply":"2022-07-21T09:26:51.505148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_models = stacking(level0_models = (lgb, xgb),\n                           level1_model = lasso)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:51.507463Z","iopub.execute_input":"2022-07-21T09:26:51.507842Z","iopub.status.idle":"2022-07-21T09:26:51.522250Z","shell.execute_reply.started":"2022-07-21T09:26:51.507810Z","shell.execute_reply":"2022-07-21T09:26:51.520953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmsle(stacked_models)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:26:51.524010Z","iopub.execute_input":"2022-07-21T09:26:51.524440Z","iopub.status.idle":"2022-07-21T09:27:01.517945Z","shell.execute_reply.started":"2022-07-21T09:26:51.524391Z","shell.execute_reply":"2022-07-21T09:27:01.517111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We do final ensembling by combining our predictions from different models by assigning different weights to each model","metadata":{}},{"cell_type":"code","source":"#prediction from xgboost model\nxgb.fit(X_train, y_train)\nxgb_train_pred = xgb.predict(X_train)\nxgb_test_pred = xgb.predict(X_test)\n\n#predictions from light gbm\nlgb.fit(X_train, y_train)\nlgb_train_pred = lgb.predict(X_train)\nlgb_test_pred = lgb.predict(X_test)\n\n#predictions from ensemble\nstacked_models.fit(X_train, y_train)\nstacked_train_pred = stacked_models.predict(X_train)\nstacked_test_pred = stacked_models.predict(X_test)\n\ny_train_pred = stacked_train_pred*0.70 + xgb_train_pred*0.15 + lgb_train_pred*0.15 \nnp.sqrt(mean_squared_error(y_train, y_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:27:01.521977Z","iopub.execute_input":"2022-07-21T09:27:01.524381Z","iopub.status.idle":"2022-07-21T09:27:05.594417Z","shell.execute_reply.started":"2022-07-21T09:27:01.524339Z","shell.execute_reply":"2022-07-21T09:27:05.593333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predictions on test set","metadata":{}},{"cell_type":"code","source":"y_test_pred = stacked_test_pred*0.70 + xgb_test_pred*0.15 + lgb_test_pred*0.15 ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T09:27:05.599384Z","iopub.execute_input":"2022-07-21T09:27:05.600091Z","iopub.status.idle":"2022-07-21T09:27:05.607236Z","shell.execute_reply.started":"2022-07-21T09:27:05.600046Z","shell.execute_reply":"2022-07-21T09:27:05.605977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submission","metadata":{}},{"cell_type":"code","source":"output = pd.DataFrame({'Id': df_test.index, 'SalePrice': np.exp(y_test_pred)})\noutput.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":0.114471,"end_time":"2022-07-19T06:52:48.605257","exception":false,"start_time":"2022-07-19T06:52:48.490786","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:27:05.608618Z","iopub.execute_input":"2022-07-21T09:27:05.609351Z","iopub.status.idle":"2022-07-21T09:27:05.627193Z","shell.execute_reply.started":"2022-07-21T09:27:05.609312Z","shell.execute_reply":"2022-07-21T09:27:05.626162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('yay')","metadata":{"papermill":{"duration":0.107424,"end_time":"2022-07-19T06:52:49.219244","exception":false,"start_time":"2022-07-19T06:52:49.111820","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-21T09:27:05.628268Z","iopub.execute_input":"2022-07-21T09:27:05.629109Z","iopub.status.idle":"2022-07-21T09:27:05.633488Z","shell.execute_reply.started":"2022-07-21T09:27:05.629066Z","shell.execute_reply":"2022-07-21T09:27:05.632624Z"},"trusted":true},"execution_count":null,"outputs":[]}]}