{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-23T19:13:47.274480Z","iopub.execute_input":"2022-07-23T19:13:47.275082Z","iopub.status.idle":"2022-07-23T19:13:47.284181Z","shell.execute_reply.started":"2022-07-23T19:13:47.275059Z","shell.execute_reply":"2022-07-23T19:13:47.283147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split,cross_validate, GridSearchCV\nfrom sklearn.preprocessing import StandardScaler,OrdinalEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import LinearRegression,Lasso,Ridge,BayesianRidge\nfrom sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\n\nimport math\nfrom IPython.display import Image\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nsns.set(rc={\"figure.figsize\": (20, 15)})\nsns.set_style(\"whitegrid\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.317548Z","iopub.execute_input":"2022-07-23T19:13:47.317978Z","iopub.status.idle":"2022-07-23T19:13:47.332043Z","shell.execute_reply.started":"2022-07-23T19:13:47.317947Z","shell.execute_reply":"2022-07-23T19:13:47.330651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Read files","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/house-prices-advanced-regression-techniques/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/house-prices-advanced-regression-techniques/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv\")\ndf = pd.concat([df_train,df_test],axis = 0,ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.334471Z","iopub.execute_input":"2022-07-23T19:13:47.334852Z","iopub.status.idle":"2022-07-23T19:13:47.405328Z","shell.execute_reply.started":"2022-07-23T19:13:47.334827Z","shell.execute_reply":"2022-07-23T19:13:47.404348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Open the text file that contains the description of various variables. Create a dictionary that takes in the key as column name and gives out description","metadata":{}},{"cell_type":"code","source":"## Create a dictionary to see the description of the column names\nwith open(\"/kaggle/input/house-prices-advanced-regression-techniques/data_description.txt\",\"r\") as f:\n        texts = f.readlines()\n\nnewlist = list()\nfor col in df.columns:\n    for text in texts:\n        if col in text:\n            newlist.append(text.split(\":\"))\n            \ndesc = dict()\nfor item in newlist:\n    if len(item)==2:\n        desc[item[0]] = item[1]\n\n## Example     \nprint(desc[\"YearRemodAdd\"])\nprint(desc[\"LandSlope\"])\nprint(desc[\"MoSold\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.420232Z","iopub.execute_input":"2022-07-23T19:13:47.420737Z","iopub.status.idle":"2022-07-23T19:13:47.441254Z","shell.execute_reply.started":"2022-07-23T19:13:47.420706Z","shell.execute_reply":"2022-07-23T19:13:47.440181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Explore Dataset","metadata":{"execution":{"iopub.status.busy":"2022-07-18T20:56:01.220407Z","iopub.execute_input":"2022-07-18T20:56:01.220740Z","iopub.status.idle":"2022-07-18T20:56:01.226097Z","shell.execute_reply.started":"2022-07-18T20:56:01.220717Z","shell.execute_reply":"2022-07-18T20:56:01.225098Z"}}},{"cell_type":"code","source":"df_train.head(10).style.background_gradient(cmap = \"viridis\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.444166Z","iopub.execute_input":"2022-07-23T19:13:47.444429Z","iopub.status.idle":"2022-07-23T19:13:47.519537Z","shell.execute_reply.started":"2022-07-23T19:13:47.444404Z","shell.execute_reply":"2022-07-23T19:13:47.518396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.520965Z","iopub.execute_input":"2022-07-23T19:13:47.521236Z","iopub.status.idle":"2022-07-23T19:13:47.560167Z","shell.execute_reply.started":"2022-07-23T19:13:47.521209Z","shell.execute_reply":"2022-07-23T19:13:47.559486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().transpose().style.background_gradient(cmap = \"magma\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.561263Z","iopub.execute_input":"2022-07-23T19:13:47.561681Z","iopub.status.idle":"2022-07-23T19:13:47.693296Z","shell.execute_reply.started":"2022-07-23T19:13:47.561647Z","shell.execute_reply":"2022-07-23T19:13:47.691977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.shape)\nprint(df_test.shape) # the SalePrice column is missing in test data, which we need to predict.","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.694627Z","iopub.execute_input":"2022-07-23T19:13:47.694975Z","iopub.status.idle":"2022-07-23T19:13:47.701682Z","shell.execute_reply.started":"2022-07-23T19:13:47.694942Z","shell.execute_reply":"2022-07-23T19:13:47.700465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var_num = [\"SalePrice\", \"OverallQual\", \"GrLivArea\", \"GarageCars\", \"TotalBsmtSF\", \"FullBath\", \"YearBuilt\"]\nsns.pairplot(df_train[var_num]);","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:47.703413Z","iopub.execute_input":"2022-07-23T19:13:47.704129Z","iopub.status.idle":"2022-07-23T19:13:57.342231Z","shell.execute_reply.started":"2022-07-23T19:13:47.704087Z","shell.execute_reply":"2022-07-23T19:13:57.340981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualize dependent variable (SalePrice)","metadata":{}},{"cell_type":"code","source":"sns.distplot(df[\"SalePrice\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:57.343868Z","iopub.execute_input":"2022-07-23T19:13:57.344167Z","iopub.status.idle":"2022-07-23T19:13:57.681540Z","shell.execute_reply.started":"2022-07-23T19:13:57.344129Z","shell.execute_reply":"2022-07-23T19:13:57.680509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We can see that the distribution is postivitely skewed.","metadata":{}},{"cell_type":"code","source":"df[\"SalePrice\"].describe()  ##There is a huge difference between the min and max..","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:57.682982Z","iopub.execute_input":"2022-07-23T19:13:57.683233Z","iopub.status.idle":"2022-07-23T19:13:57.694428Z","shell.execute_reply.started":"2022-07-23T19:13:57.683210Z","shell.execute_reply":"2022-07-23T19:13:57.692952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"LogSalePrice\"] = np.log10(df[\"SalePrice\"])\nsns.distplot(df[\"LogSalePrice\"],color = 'r')","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:57.696029Z","iopub.execute_input":"2022-07-23T19:13:57.696490Z","iopub.status.idle":"2022-07-23T19:13:58.058947Z","shell.execute_reply.started":"2022-07-23T19:13:57.696433Z","shell.execute_reply":"2022-07-23T19:13:58.057550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Since, the target variable is positively skewed, we would like to transform the variable using log of the price, which gave us a normally distributed chart, which is better for machine learning prediction","metadata":{}},{"cell_type":"markdown","source":"### Question 1: What features correlates the most with the Sale Price of the house?","metadata":{}},{"cell_type":"code","source":"## Lets create a list of column names for categorical and numerical features\n\ncate_feat = list(df.select_dtypes(include = [object]).columns)\nnum_feat = list(df.select_dtypes(include = [int,float]).columns)\n\nprint(cate_feat)\nprint('\\n')\nprint(num_feat)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:58.060380Z","iopub.execute_input":"2022-07-23T19:13:58.060775Z","iopub.status.idle":"2022-07-23T19:13:58.072074Z","shell.execute_reply.started":"2022-07-23T19:13:58.060744Z","shell.execute_reply":"2022-07-23T19:13:58.070812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Heatmap for all the remaining numerical data including the taget 'SalePrice'\n# Define the heatmap parameters\npd.options.display.float_format = \"{:,.2f}\".format\n\n# Define correlation matrix\ncorr_matrix = df[num_feat].corr()\n\n# Replace correlation < |0.3| by 0 for a better visibility\ncorr_matrix[(corr_matrix < 0.3) & (corr_matrix > -0.3)] = 0\n\n# plot the heatmap\nsns.heatmap(corr_matrix, vmax=1.0, vmin=-1.0, linewidths=0.1,\n            annot_kws={\"size\": 9, \"color\": \"black\"},annot=True)\nplt.title(\"SalePrice Correlation\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:13:58.076663Z","iopub.execute_input":"2022-07-23T19:13:58.076922Z","iopub.status.idle":"2022-07-23T19:14:01.976017Z","shell.execute_reply.started":"2022-07-23T19:13:58.076900Z","shell.execute_reply":"2022-07-23T19:14:01.975234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We can see that, The overall quality of the house and Ground living Area has 0.79 and 0.74 correlation with the SalePrice of the house, followed by Garagecars with 0.64 correlation.","metadata":{}},{"cell_type":"code","source":"## Lets visualize individually \n\ncorr =df.corr()[\"SalePrice\"].sort_values(ascending = False)[2:8] ## selecting cols other than Saleprice, LogPrice\ncorr","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:01.977110Z","iopub.execute_input":"2022-07-23T19:14:01.977775Z","iopub.status.idle":"2022-07-23T19:14:01.997254Z","shell.execute_reply.started":"2022-07-23T19:14:01.977747Z","shell.execute_reply":"2022-07-23T19:14:01.996469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f,ax = plt.subplots(nrows = 6,ncols = 1, figsize = (20,40))\nfor i,col in enumerate(corr.index):    \n    sns.scatterplot(x = col, y = \"SalePrice\", data = df, ax = ax[i], color = 'darkorange')\n    ax[i].set_title(f'{col} vs SalePrice')\n   ","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:01.998292Z","iopub.execute_input":"2022-07-23T19:14:01.998538Z","iopub.status.idle":"2022-07-23T19:14:03.270671Z","shell.execute_reply.started":"2022-07-23T19:14:01.998515Z","shell.execute_reply":"2022-07-23T19:14:03.269218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We can see the features above are quite linear to the Sale Price suggesting positive correlation.","metadata":{}},{"cell_type":"markdown","source":"### Question 1: What year were most of the houses built (Top 10), and does the year built say anything regarding the sale price?","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=\"YearBuilt\", y=\"SalePrice\", data=df,)\nfig.axis(ymin=0, ymax=900000);\nplt.xticks(rotation=90);\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:03.272246Z","iopub.execute_input":"2022-07-23T19:14:03.272524Z","iopub.status.idle":"2022-07-23T19:14:06.475965Z","shell.execute_reply.started":"2022-07-23T19:14:03.272497Z","shell.execute_reply":"2022-07-23T19:14:06.475063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yr_built = pd.DataFrame({\"Count\":df[\"YearBuilt\"].value_counts()[:10]}).reset_index()\nyr_built.rename(columns={'index':'Year'},inplace=True)\nplt.figure(figsize = (20,10))\nsns.barplot(x = 'Year', y = \"Count\", data = yr_built)\nplt.title(\"Year Built\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:06.477902Z","iopub.execute_input":"2022-07-23T19:14:06.478265Z","iopub.status.idle":"2022-07-23T19:14:06.690153Z","shell.execute_reply.started":"2022-07-23T19:14:06.478240Z","shell.execute_reply":"2022-07-23T19:14:06.688943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## As a side question, lets see if there is a huge difference in sale price based on different months\n\ndf.groupby(\"MoSold\").mean()[\"SalePrice\"].sort_values(ascending = False).plot(kind = 'bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:06.691534Z","iopub.execute_input":"2022-07-23T19:14:06.691809Z","iopub.status.idle":"2022-07-23T19:14:06.950139Z","shell.execute_reply.started":"2022-07-23T19:14:06.691782Z","shell.execute_reply":"2022-07-23T19:14:06.949059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Most of the houses were built in the year 2005, 2006. On average the houses built after 1980 have higher Saleprice. There is also no significance difference in terms of average saleprice based on month. September taking the lead.","metadata":{}},{"cell_type":"markdown","source":"## Handling Missing Data","metadata":{}},{"cell_type":"code","source":"## Number of missing values in categorical features\ndf[cate_feat].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:06.951106Z","iopub.execute_input":"2022-07-23T19:14:06.951318Z","iopub.status.idle":"2022-07-23T19:14:06.968464Z","shell.execute_reply.started":"2022-07-23T19:14:06.951297Z","shell.execute_reply":"2022-07-23T19:14:06.967300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Number of missing values in numerical features\ndf[num_feat].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:06.970564Z","iopub.execute_input":"2022-07-23T19:14:06.971198Z","iopub.status.idle":"2022-07-23T19:14:06.981997Z","shell.execute_reply.started":"2022-07-23T19:14:06.971160Z","shell.execute_reply":"2022-07-23T19:14:06.981000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dealing with null values in numerical features","metadata":{}},{"cell_type":"markdown","source":"##### Since there is alot missing in lotfrontage, and is related to lotArea as seen below, we will use linear reg to fill in the missing values for LotFrontage","metadata":{}},{"cell_type":"code","source":"sns.lmplot(x=\"LotArea\",y=\"LotFrontage\",data = df)\nplt.ylabel(\"LotFrontage\")\nplt.xlabel(\"LotArea\")\nplt.title(\"LotArea vs LotFrontage\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:06.983641Z","iopub.execute_input":"2022-07-23T19:14:06.984228Z","iopub.status.idle":"2022-07-23T19:14:07.679035Z","shell.execute_reply.started":"2022-07-23T19:14:06.984192Z","shell.execute_reply":"2022-07-23T19:14:07.677748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lm = LinearRegression()\nlm_X = df[df['LotFrontage'].notnull()]['LotArea'].values.reshape(-1,1)\nlm_y = df[df['LotFrontage'].notnull()]['LotFrontage'].values\nlm.fit(lm_X,lm_y)\ndf['LotFrontage'].fillna((df['LotArea'] * lm.coef_[0] + lm.intercept_), inplace=True)\ndf['LotFrontage'] = df['LotFrontage'].apply(lambda x: int(x))","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.680713Z","iopub.execute_input":"2022-07-23T19:14:07.681056Z","iopub.status.idle":"2022-07-23T19:14:07.710662Z","shell.execute_reply.started":"2022-07-23T19:14:07.681020Z","shell.execute_reply":"2022-07-23T19:14:07.709585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Since its only few values missing in each category, we dont need to explore alot. Just filling na values with median and mean","metadata":{}},{"cell_type":"code","source":"df[\"GarageYrBlt\"].fillna(df[\"GarageYrBlt\"].median(),inplace = True)\ndf[\"BsmtFinSF1\"].fillna(df[\"BsmtFinSF1\"].mean(),inplace = True)\ndf[\"BsmtFinSF2\"].fillna(df[\"BsmtFinSF2\"].mean(),inplace = True)\ndf[\"BsmtUnfSF\"].fillna(df[\"BsmtUnfSF\"].mean(),inplace = True)\ndf[\"TotalBsmtSF\"].fillna(df[\"TotalBsmtSF\"].mean(),inplace = True)\ndf[\"BsmtFullBath\"].fillna(df[\"BsmtFullBath\"].median(),inplace = True)\ndf[\"BsmtHalfBath\"].fillna(df[\"BsmtHalfBath\"].median(),inplace = True)\ndf[\"GarageArea\"].fillna(df[\"GarageArea\"].mean(),inplace = True)\ndf[\"GarageCars\"].fillna(int(df[\"GarageCars\"].median()),inplace = True)\ndf[\"MasVnrArea\"].fillna(df[\"MasVnrArea\"].median(),inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.712000Z","iopub.execute_input":"2022-07-23T19:14:07.712756Z","iopub.status.idle":"2022-07-23T19:14:07.727219Z","shell.execute_reply.started":"2022-07-23T19:14:07.712728Z","shell.execute_reply":"2022-07-23T19:14:07.726168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dealing with null values in Categorical features","metadata":{}},{"cell_type":"code","source":"## Since more than 90% of the column is null\ndf.drop([\"Alley\",\"FireplaceQu\",\"PoolQC\",\"MiscFeature\"], axis = 1,inplace = True)\ndf.drop([\"Fence\"], axis = 1,inplace = True)\n\n# Remove the items from the column list\nfor item in cate_feat:\n    if item in [\"Alley\",\"FireplaceQu\",\"PoolQC\",\"MiscFeature\"]:\n        cate_feat.remove(item) \ncate_feat.remove(\"Fence\")   \n\n# Fill null values with none for items that are missing the item and mode for the rest of the missing values\ncate_none = [\"BsmtExposure\", \"BsmtFinType2\", \"BsmtCond\", \"BsmtQual\", \"BsmtFinType1\"]\ncate_mode = [\"Electrical\", \"Functional\", \"KitchenQual\", \"Exterior1st\", \"Exterior2nd\", \"MSZoning\", \"SaleType\", \"MasVnrType\", \"GarageFinish\", \"GarageQual\", \"GarageCond\", \"GarageType\",\"Utilities\"]\n\nfor col in cate_none:\n    df[col].fillna('none',inplace = True)\n\nfor col in cate_mode:\n    df[col].fillna(df[col].mode()[0],inplace = True)\n\nprint(f\"Null values: {df.drop(['SalePrice','LogSalePrice'],axis = 1).isnull().sum().sum()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.728327Z","iopub.execute_input":"2022-07-23T19:14:07.728708Z","iopub.status.idle":"2022-07-23T19:14:07.780833Z","shell.execute_reply.started":"2022-07-23T19:14:07.728673Z","shell.execute_reply":"2022-07-23T19:14:07.779744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image(\"/kaggle/input/datatype-pic/types-of-data--1024x555.png\")","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.781866Z","iopub.execute_input":"2022-07-23T19:14:07.783099Z","iopub.status.idle":"2022-07-23T19:14:07.795072Z","shell.execute_reply.started":"2022-07-23T19:14:07.783068Z","shell.execute_reply":"2022-07-23T19:14:07.794153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical Variable Encoding","metadata":{}},{"cell_type":"markdown","source":"#### It is very important to understand the data-type while analyzing the dataset. Since proper grouping of data will give better predictive results. There are 4 types of Data, namely Discrete, Continuous, Nominal and Ordinal. For categorical data, we will be dealing with Nominal and Ordinal.","metadata":{}},{"cell_type":"markdown","source":"#### -ordinal: There is a clear order in the category, example: poor, good, excellent etc\n#### -Nominal: These are usually names without and order","metadata":{}},{"cell_type":"markdown","source":"#### We will be encoding categories using 3 different methods. \n#### - ordinal grouping (for ordinal category)\n#### - Get dummies (for categories with less than 8(upto you) unique items)\n#### - Top (8-10) freq occuring item within a category ","metadata":{}},{"cell_type":"code","source":"## Lets create a dataframe of category with its unique features\n\nunq_col = dict()\nfor col in cate_feat:\n    unq_col[col] = list(df[col].unique())\n\nunq_df = pd.DataFrame.from_dict(unq_col, orient=\"index\").replace({None:0})\nunq_df","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.796270Z","iopub.execute_input":"2022-07-23T19:14:07.797089Z","iopub.status.idle":"2022-07-23T19:14:07.866193Z","shell.execute_reply.started":"2022-07-23T19:14:07.797061Z","shell.execute_reply":"2022-07-23T19:14:07.864923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Deealing with the Ordinal var first, These rep some kind of order\ncate1 = [\"BsmtCond\"]\ncate1_item = ['none',\"Po\", \"Fa\", \"TA\", \"Gd\"]\n\ncate2 = [\"BsmtExposure\"]\ncate2_item = ['none','No','Mn','Av','Gd']\n\ncate3 = [\"BsmtQual\"]\ncate3_item = ['none',\"Fa\",\"TA\",\"Gd\", \"Ex\"]\n\ncate4 = [\"ExterCond\", \"HeatingQC\"]\ncate4_item = [\"Po\", \"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\ncate5 = [\"ExterQual\", \"KitchenQual\"]\ncate5_item = [\"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\ncate6 = [\"GarageQual\", \"GarageCond\"]\ncate6_item = ['none',\"Po\", \"Fa\", \"TA\", \"Gd\", \"Ex\"]\n\ncate7 = [\"BsmtFinType1\", \"BsmtFinType2\"]\ncate7_item = ['none',\"Unf\", \"LwQ\", \"Rec\", \"BLQ\", \"ALQ\", \"GLQ\"]\n\ncate = [cate1,cate2,cate3,cate4,cate5,cate6,cate7]\ncate_item = [cate1_item,cate2_item,cate3_item,cate4_item,cate5_item,cate6_item,cate7_item]","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.867605Z","iopub.execute_input":"2022-07-23T19:14:07.867862Z","iopub.status.idle":"2022-07-23T19:14:07.878165Z","shell.execute_reply.started":"2022-07-23T19:14:07.867836Z","shell.execute_reply":"2022-07-23T19:14:07.876411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Takes the items in a category and converts into numerical value ( with order respected)\nfor idx in range(len(cate)):\n    encoder = OrdinalEncoder(categories = [cate_item[idx]])\n    \n    for col in cate[idx]:\n        df[col] = encoder.fit_transform(df[[col]])","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.880715Z","iopub.execute_input":"2022-07-23T19:14:07.881317Z","iopub.status.idle":"2022-07-23T19:14:07.931998Z","shell.execute_reply.started":"2022-07-23T19:14:07.881282Z","shell.execute_reply":"2022-07-23T19:14:07.930839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### One-Hot Encoding","metadata":{}},{"cell_type":"code","source":"cate_ord = cate1+cate2+cate3+cate4+cate5+cate6+cate7\n\ncate_one_hot = list()\ncate_target_var = list()\n\nfor col in df[cate_feat].drop(cate_ord,axis = 1).columns:\n    if len(df[col].unique()) <6:\n        cate_one_hot.append(col)\n    else:\n        cate_target_var.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.934042Z","iopub.execute_input":"2022-07-23T19:14:07.934432Z","iopub.status.idle":"2022-07-23T19:14:07.956639Z","shell.execute_reply.started":"2022-07-23T19:14:07.934390Z","shell.execute_reply":"2022-07-23T19:14:07.955720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummies_one_hot = pd.get_dummies(df[cate_one_hot], drop_first = True)\ndummies_one_hot","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:07.960257Z","iopub.execute_input":"2022-07-23T19:14:07.960538Z","iopub.status.idle":"2022-07-23T19:14:08.003048Z","shell.execute_reply.started":"2022-07-23T19:14:07.960511Z","shell.execute_reply":"2022-07-23T19:14:08.002297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Finds the top 6 item in each category and creates a one hot encoding. Done on categories with alot\n## of items to reduce the dimensions.\ndef one_hot(df):\n    \n    for col in df:\n        top_10 = [item for item in df[col].value_counts().sort_values(ascending = False).head(6).index]\n        \n        for label in top_10:\n            df[label] = np.where(df[col]==label,1,0)\n            \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.004136Z","iopub.execute_input":"2022-07-23T19:14:08.004382Z","iopub.status.idle":"2022-07-23T19:14:08.010754Z","shell.execute_reply.started":"2022-07-23T19:14:08.004357Z","shell.execute_reply":"2022-07-23T19:14:08.009991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One hot encoding for nominal categorical data\ndf_tar_var = one_hot(df[cate_target_var])\ndf_tar_var.drop(cate_target_var,axis = 1,inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.011357Z","iopub.execute_input":"2022-07-23T19:14:08.011592Z","iopub.status.idle":"2022-07-23T19:14:08.119481Z","shell.execute_reply.started":"2022-07-23T19:14:08.011571Z","shell.execute_reply":"2022-07-23T19:14:08.118746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creating a dataframe with all converted values for prediction\ndf_final = pd.concat([df_tar_var,dummies_one_hot,df],axis = 1)\ndf_final.drop(cate_feat+cate_ord,axis = 1, inplace = True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.120359Z","iopub.execute_input":"2022-07-23T19:14:08.120625Z","iopub.status.idle":"2022-07-23T19:14:08.135981Z","shell.execute_reply.started":"2022-07-23T19:14:08.120600Z","shell.execute_reply":"2022-07-23T19:14:08.135097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.137319Z","iopub.execute_input":"2022-07-23T19:14:08.137589Z","iopub.status.idle":"2022-07-23T19:14:08.158365Z","shell.execute_reply.started":"2022-07-23T19:14:08.137565Z","shell.execute_reply":"2022-07-23T19:14:08.157574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"code","source":"## Divide the dataset into train and test\n\ntrain_df = df_final[df_final[\"SalePrice\"].notnull()]\ntest_df = df_final[df_final[\"SalePrice\"].isnull()]\n\nprint(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.159693Z","iopub.execute_input":"2022-07-23T19:14:08.160281Z","iopub.status.idle":"2022-07-23T19:14:08.170015Z","shell.execute_reply.started":"2022-07-23T19:14:08.160251Z","shell.execute_reply":"2022-07-23T19:14:08.169004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Split the dataset into X and Y\nX_train = train_df.drop([\"SalePrice\",\"LogSalePrice\"],axis = 1)\ny_train = train_df[\"LogSalePrice\"]\nX_test = test_df.drop([\"SalePrice\",\"LogSalePrice\"],axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.171275Z","iopub.execute_input":"2022-07-23T19:14:08.171589Z","iopub.status.idle":"2022-07-23T19:14:08.180473Z","shell.execute_reply.started":"2022-07-23T19:14:08.171552Z","shell.execute_reply":"2022-07-23T19:14:08.179715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Scale the dataset\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.181665Z","iopub.execute_input":"2022-07-23T19:14:08.182383Z","iopub.status.idle":"2022-07-23T19:14:08.205088Z","shell.execute_reply.started":"2022-07-23T19:14:08.182354Z","shell.execute_reply":"2022-07-23T19:14:08.204149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Selection","metadata":{"execution":{"iopub.status.busy":"2022-07-23T14:47:32.253208Z","iopub.execute_input":"2022-07-23T14:47:32.253794Z","iopub.status.idle":"2022-07-23T14:47:32.263741Z","shell.execute_reply.started":"2022-07-23T14:47:32.253718Z","shell.execute_reply":"2022-07-23T14:47:32.261444Z"}}},{"cell_type":"markdown","source":"#### Cross validation to find the best predicting model","metadata":{}},{"cell_type":"code","source":"model = {\n    'Lasso' : Lasso(),\n    'Ridge' : Ridge(),\n    'XGB' : XGBRegressor(),\n    'LGBM' : LGBMRegressor(),\n    'Gradient Boosting' : GradientBoostingRegressor(),\n    'Bayesian Ridge' : BayesianRidge()\n}\n\ndf_result = pd.DataFrame(columns =[\"Model_name\",\"RMSE\"])\n\nfor name,mod in model.items():\n    \n    cross_val = cross_validate(mod,X = X_train,y = y_train, cv = 10, scoring = (['neg_root_mean_squared_error']))\n    \n    df_result= df_result.append({'Model_name':name,\"RMSE\": np.abs(cross_val['test_neg_root_mean_squared_error']).mean()}, ignore_index = True)\n    \ndf_result = df_result.sort_values('RMSE', ascending=True)\n\ndf_result","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:08.206314Z","iopub.execute_input":"2022-07-23T19:14:08.206938Z","iopub.status.idle":"2022-07-23T19:14:26.115802Z","shell.execute_reply.started":"2022-07-23T19:14:08.206905Z","shell.execute_reply":"2022-07-23T19:14:26.114833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We will choose Gradient Boosting an LGBM as it gave the best result for hyperparameter tuning.","metadata":{}},{"cell_type":"markdown","source":"#### GradientBoostingRegressor","metadata":{}},{"cell_type":"code","source":"gb = GradientBoostingRegressor()\nparams_gb = {\n    'loss' : ('squared_error', 'absolute_error','huber'),\n    'learning_rate' : (1.0, 0.1, 0.01),\n    'n_estimators' : (100, 200, 300)\n}\n\nmod_gb = GridSearchCV(gb, params_gb, cv=10)\nmod_gb.fit(X_train, y_train)\nprint('Best_hyperparameter : ', mod_gb.best_params_)\n\npred_gb = mod_gb.predict(X_train)\nprint(f'RMSE : {mean_squared_error(y_train,pred_gb, squared=False)}')\n    \n    \n","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:14:26.118262Z","iopub.execute_input":"2022-07-23T19:14:26.118614Z","iopub.status.idle":"2022-07-23T19:23:31.245403Z","shell.execute_reply.started":"2022-07-23T19:14:26.118577Z","shell.execute_reply":"2022-07-23T19:23:31.244186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### LGBMRegressor","metadata":{}},{"cell_type":"code","source":"lgbm = LGBMRegressor()\nparams_lgbm = {\n    'num_leaves' : (11, 31, 41),\n    'learning_rate' : (0.5, 0.1, 0.05),\n    'n_estimators' : (100, 200, 300)\n}\n\nmod_lgbm = GridSearchCV(lgbm, params_lgbm, cv=10)\nmod_lgbm.fit(X_train, y_train)\nprint('Best_hyperparameter : ', mod_lgbm.best_params_)\n\npred_lgbm = mod_lgbm.predict(X_train)\nprint(f'RMSE : {mean_squared_error(y_train, pred_lgbm, squared=False)}')","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:23:31.251411Z","iopub.execute_input":"2022-07-23T19:23:31.251948Z","iopub.status.idle":"2022-07-23T19:25:16.753743Z","shell.execute_reply.started":"2022-07-23T19:23:31.251923Z","shell.execute_reply":"2022-07-23T19:25:16.752928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict the test dataset","metadata":{}},{"cell_type":"code","source":"## Predict the test data and inverse log\ny_pred = mod_gb.predict(X_test)\ny_pred_inv = 10 ** y_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:25:16.755177Z","iopub.execute_input":"2022-07-23T19:25:16.755737Z","iopub.status.idle":"2022-07-23T19:25:16.767157Z","shell.execute_reply.started":"2022-07-23T19:25:16.755706Z","shell.execute_reply":"2022-07-23T19:25:16.766301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"submission['SalePrice'] = y_pred_inv\nsubmission.to_csv('final_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-23T19:25:16.768737Z","iopub.execute_input":"2022-07-23T19:25:16.769211Z","iopub.status.idle":"2022-07-23T19:25:16.783721Z","shell.execute_reply.started":"2022-07-23T19:25:16.769181Z","shell.execute_reply":"2022-07-23T19:25:16.782863Z"},"trusted":true},"execution_count":null,"outputs":[]}]}