{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# House Price Prediction\n\n### Reading the description of dataset\n### Reading and Understanding the dataset\n\n### Data Cleaning\n#### * Missing Values\n#### * Datatype correction\n\n### EDA\n#### * Univariate Analysis\n#### * BiVariate Analysis\n#### * Correlation\n#### * Outlier Analysis\n\n### Data Pre-Processing\n#### * Encoding the categorical features\n#### * Handling biased target variable\n#### * Splitting the dataset\n#### * Scaling\n\n### Model Building\n#### * Statsmodel OLS\n#### * Recursive Feature Elimination\n#### * Manual Feature Elimination\n#### * Linear Regression with Assumptions\n#### * Random Forest Regressor\n#### * Hypertuning of RF parameters\n#### * Gradient Boosting Regressor\n#### * Hypertuning GBM parameters\n#### * Adaboost \n#### * Hypertuning AdaBoost\n\n### Results","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-07-21T11:43:33.878270Z","iopub.execute_input":"2022-07-21T11:43:33.878826Z","iopub.status.idle":"2022-07-21T11:43:33.891300Z","shell.execute_reply.started":"2022-07-21T11:43:33.878774Z","shell.execute_reply":"2022-07-21T11:43:33.890234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading the description of file","metadata":{}},{"cell_type":"code","source":"descr= open(r\"../input/house-prices-advanced-regression-techniques/data_description.txt\", \"r\")\ndescr.readlines()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:33.893639Z","iopub.execute_input":"2022-07-21T11:43:33.894438Z","iopub.status.idle":"2022-07-21T11:43:33.911396Z","shell.execute_reply.started":"2022-07-21T11:43:33.894399Z","shell.execute_reply":"2022-07-21T11:43:33.910259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Importing relevant libraries\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:33.912916Z","iopub.execute_input":"2022-07-21T11:43:33.913306Z","iopub.status.idle":"2022-07-21T11:43:33.918991Z","shell.execute_reply.started":"2022-07-21T11:43:33.913276Z","shell.execute_reply":"2022-07-21T11:43:33.918220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading and understanding the dataset","metadata":{}},{"cell_type":"code","source":"df_train= pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:33.920181Z","iopub.execute_input":"2022-07-21T11:43:33.920952Z","iopub.status.idle":"2022-07-21T11:43:34.008100Z","shell.execute_reply.started":"2022-07-21T11:43:33.920923Z","shell.execute_reply":"2022-07-21T11:43:34.007195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test= pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.010789Z","iopub.execute_input":"2022-07-21T11:43:34.011317Z","iopub.status.idle":"2022-07-21T11:43:34.091591Z","shell.execute_reply.started":"2022-07-21T11:43:34.011286Z","shell.execute_reply":"2022-07-21T11:43:34.090460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.092965Z","iopub.execute_input":"2022-07-21T11:43:34.093286Z","iopub.status.idle":"2022-07-21T11:43:34.123063Z","shell.execute_reply.started":"2022-07-21T11:43:34.093258Z","shell.execute_reply":"2022-07-21T11:43:34.121905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 38 numerical columns and 43 object columns","metadata":{}},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.124783Z","iopub.execute_input":"2022-07-21T11:43:34.125416Z","iopub.status.idle":"2022-07-21T11:43:34.132887Z","shell.execute_reply.started":"2022-07-21T11:43:34.125377Z","shell.execute_reply":"2022-07-21T11:43:34.132039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Cleaning","metadata":{}},{"cell_type":"code","source":"#Handling the missing values\nmissing= df_train.isnull().sum()[df_train.isnull().sum() >0].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.134325Z","iopub.execute_input":"2022-07-21T11:43:34.134834Z","iopub.status.idle":"2022-07-21T11:43:34.161813Z","shell.execute_reply.started":"2022-07-21T11:43:34.134803Z","shell.execute_reply":"2022-07-21T11:43:34.160066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Checking the percentage of missing values\n(missing/df_train.shape[0])*100","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.163863Z","iopub.execute_input":"2022-07-21T11:43:34.165435Z","iopub.status.idle":"2022-07-21T11:43:34.174917Z","shell.execute_reply.started":"2022-07-21T11:43:34.165369Z","shell.execute_reply":"2022-07-21T11:43:34.174189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dropping the columns having missing values greater than 45%\ncol_to_drop= list(missing[(missing/df_train.shape[0])*100 > 40.0].index)\ncol_to_drop","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.176111Z","iopub.execute_input":"2022-07-21T11:43:34.177087Z","iopub.status.idle":"2022-07-21T11:43:34.191267Z","shell.execute_reply.started":"2022-07-21T11:43:34.177056Z","shell.execute_reply":"2022-07-21T11:43:34.190415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.drop(col_to_drop, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.192984Z","iopub.execute_input":"2022-07-21T11:43:34.193888Z","iopub.status.idle":"2022-07-21T11:43:34.204916Z","shell.execute_reply.started":"2022-07-21T11:43:34.193836Z","shell.execute_reply":"2022-07-21T11:43:34.203894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop(col_to_drop, axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.207035Z","iopub.execute_input":"2022-07-21T11:43:34.207441Z","iopub.status.idle":"2022-07-21T11:43:34.220041Z","shell.execute_reply.started":"2022-07-21T11:43:34.207397Z","shell.execute_reply":"2022-07-21T11:43:34.218661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imputing the missing values\n# LotFrontage: Linear feet of street connected to property\ndf_train.LotFrontage.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.221759Z","iopub.execute_input":"2022-07-21T11:43:34.222832Z","iopub.status.idle":"2022-07-21T11:43:34.237200Z","shell.execute_reply.started":"2022-07-21T11:43:34.222788Z","shell.execute_reply":"2022-07-21T11:43:34.235942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Statistics of the variable\ndf_train.LotFrontage.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.241763Z","iopub.execute_input":"2022-07-21T11:43:34.242335Z","iopub.status.idle":"2022-07-21T11:43:34.252446Z","shell.execute_reply.started":"2022-07-21T11:43:34.242302Z","shell.execute_reply":"2022-07-21T11:43:34.251584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(df_train.LotFrontage)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.253760Z","iopub.execute_input":"2022-07-21T11:43:34.254143Z","iopub.status.idle":"2022-07-21T11:43:34.425904Z","shell.execute_reply.started":"2022-07-21T11:43:34.254100Z","shell.execute_reply":"2022-07-21T11:43:34.425129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data seems to be right skewed and also there are some outliers present in the variable as observed from boxplot. So imputing the median values.","metadata":{}},{"cell_type":"code","source":"df_train.LotFrontage = df_train.LotFrontage.fillna(df_train.LotFrontage.median())\ndf_train.LotFrontage.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.429315Z","iopub.execute_input":"2022-07-21T11:43:34.429639Z","iopub.status.idle":"2022-07-21T11:43:34.439189Z","shell.execute_reply.started":"2022-07-21T11:43:34.429610Z","shell.execute_reply":"2022-07-21T11:43:34.438152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('LotFrontage', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.442825Z","iopub.execute_input":"2022-07-21T11:43:34.443298Z","iopub.status.idle":"2022-07-21T11:43:34.453809Z","shell.execute_reply.started":"2022-07-21T11:43:34.443262Z","shell.execute_reply":"2022-07-21T11:43:34.452954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GarageType\ndf_train.GarageType.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.454862Z","iopub.execute_input":"2022-07-21T11:43:34.456200Z","iopub.status.idle":"2022-07-21T11:43:34.466652Z","shell.execute_reply.started":"2022-07-21T11:43:34.456137Z","shell.execute_reply":"2022-07-21T11:43:34.465488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#It is a categorical feature, hence imputing the mode value\ndf_train.GarageType = df_train.GarageType.fillna(df_train.GarageType.mode()[0])\ndf_train.GarageType.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.468307Z","iopub.execute_input":"2022-07-21T11:43:34.468841Z","iopub.status.idle":"2022-07-21T11:43:34.481619Z","shell.execute_reply.started":"2022-07-21T11:43:34.468811Z","shell.execute_reply":"2022-07-21T11:43:34.480383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('GarageType', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.482906Z","iopub.execute_input":"2022-07-21T11:43:34.484022Z","iopub.status.idle":"2022-07-21T11:43:34.498586Z","shell.execute_reply.started":"2022-07-21T11:43:34.483979Z","shell.execute_reply":"2022-07-21T11:43:34.497489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GarageYrBlt\ndf_train['GarageYrBlt'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.500262Z","iopub.execute_input":"2022-07-21T11:43:34.501031Z","iopub.status.idle":"2022-07-21T11:43:34.510488Z","shell.execute_reply.started":"2022-07-21T11:43:34.500984Z","shell.execute_reply":"2022-07-21T11:43:34.509533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.GarageYrBlt= df_train.GarageYrBlt.fillna(df_train.GarageYrBlt.median())\ndf_train.GarageYrBlt.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.512055Z","iopub.execute_input":"2022-07-21T11:43:34.512391Z","iopub.status.idle":"2022-07-21T11:43:34.526203Z","shell.execute_reply.started":"2022-07-21T11:43:34.512362Z","shell.execute_reply":"2022-07-21T11:43:34.525425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('GarageYrBlt', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.527188Z","iopub.execute_input":"2022-07-21T11:43:34.528022Z","iopub.status.idle":"2022-07-21T11:43:34.540815Z","shell.execute_reply.started":"2022-07-21T11:43:34.527987Z","shell.execute_reply":"2022-07-21T11:43:34.540061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GarageFinish\ndf_train.GarageFinish.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.541879Z","iopub.execute_input":"2022-07-21T11:43:34.542714Z","iopub.status.idle":"2022-07-21T11:43:34.555334Z","shell.execute_reply.started":"2022-07-21T11:43:34.542681Z","shell.execute_reply":"2022-07-21T11:43:34.554494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.GarageFinish = df_train.GarageFinish.fillna(df_train.GarageFinish.mode()[0])\ndf_train.GarageFinish.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.556628Z","iopub.execute_input":"2022-07-21T11:43:34.557437Z","iopub.status.idle":"2022-07-21T11:43:34.575397Z","shell.execute_reply.started":"2022-07-21T11:43:34.557402Z","shell.execute_reply":"2022-07-21T11:43:34.574520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('GarageFinish', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.576509Z","iopub.execute_input":"2022-07-21T11:43:34.576962Z","iopub.status.idle":"2022-07-21T11:43:34.591815Z","shell.execute_reply.started":"2022-07-21T11:43:34.576934Z","shell.execute_reply":"2022-07-21T11:43:34.590641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#GarageQual\ndf_train.GarageQual.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.593229Z","iopub.execute_input":"2022-07-21T11:43:34.594130Z","iopub.status.idle":"2022-07-21T11:43:34.607773Z","shell.execute_reply.started":"2022-07-21T11:43:34.594097Z","shell.execute_reply":"2022-07-21T11:43:34.606843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.GarageQual= df_train.GarageQual.fillna(df_train.GarageQual.mode()[0])\ndf_train.GarageQual.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.609424Z","iopub.execute_input":"2022-07-21T11:43:34.609991Z","iopub.status.idle":"2022-07-21T11:43:34.620387Z","shell.execute_reply.started":"2022-07-21T11:43:34.609923Z","shell.execute_reply":"2022-07-21T11:43:34.619530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('GarageQual', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.621668Z","iopub.execute_input":"2022-07-21T11:43:34.622193Z","iopub.status.idle":"2022-07-21T11:43:34.638774Z","shell.execute_reply.started":"2022-07-21T11:43:34.622140Z","shell.execute_reply":"2022-07-21T11:43:34.637735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.GarageCond.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.640058Z","iopub.execute_input":"2022-07-21T11:43:34.640663Z","iopub.status.idle":"2022-07-21T11:43:34.653439Z","shell.execute_reply.started":"2022-07-21T11:43:34.640633Z","shell.execute_reply":"2022-07-21T11:43:34.652580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.GarageCond= df_train.GarageCond.fillna(df_train.GarageCond.mode()[0])\ndf_train.GarageCond.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.654638Z","iopub.execute_input":"2022-07-21T11:43:34.655379Z","iopub.status.idle":"2022-07-21T11:43:34.666909Z","shell.execute_reply.started":"2022-07-21T11:43:34.655345Z","shell.execute_reply":"2022-07-21T11:43:34.666041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('GarageCond', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.668473Z","iopub.execute_input":"2022-07-21T11:43:34.668948Z","iopub.status.idle":"2022-07-21T11:43:34.677982Z","shell.execute_reply.started":"2022-07-21T11:43:34.668914Z","shell.execute_reply":"2022-07-21T11:43:34.677055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtExposure.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.679173Z","iopub.execute_input":"2022-07-21T11:43:34.679932Z","iopub.status.idle":"2022-07-21T11:43:34.692000Z","shell.execute_reply.started":"2022-07-21T11:43:34.679892Z","shell.execute_reply":"2022-07-21T11:43:34.690710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtExposure = df_train.BsmtExposure.fillna(df_train.BsmtExposure.mode()[0])\ndf_train.BsmtExposure.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.693271Z","iopub.execute_input":"2022-07-21T11:43:34.693731Z","iopub.status.idle":"2022-07-21T11:43:34.703432Z","shell.execute_reply.started":"2022-07-21T11:43:34.693653Z","shell.execute_reply":"2022-07-21T11:43:34.702243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('BsmtExposure', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.704832Z","iopub.execute_input":"2022-07-21T11:43:34.705410Z","iopub.status.idle":"2022-07-21T11:43:34.715395Z","shell.execute_reply.started":"2022-07-21T11:43:34.705379Z","shell.execute_reply":"2022-07-21T11:43:34.714298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtFinType2.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.716785Z","iopub.execute_input":"2022-07-21T11:43:34.717365Z","iopub.status.idle":"2022-07-21T11:43:34.726617Z","shell.execute_reply.started":"2022-07-21T11:43:34.717304Z","shell.execute_reply":"2022-07-21T11:43:34.725430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtFinType2= df_train.BsmtFinType2.fillna(df_train.BsmtFinType2.mode()[0])\ndf_train.BsmtFinType2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.728174Z","iopub.execute_input":"2022-07-21T11:43:34.728820Z","iopub.status.idle":"2022-07-21T11:43:34.739897Z","shell.execute_reply.started":"2022-07-21T11:43:34.728779Z","shell.execute_reply":"2022-07-21T11:43:34.738579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('BsmtFinType2', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.743208Z","iopub.execute_input":"2022-07-21T11:43:34.743912Z","iopub.status.idle":"2022-07-21T11:43:34.754752Z","shell.execute_reply.started":"2022-07-21T11:43:34.743877Z","shell.execute_reply":"2022-07-21T11:43:34.753817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtFinType1.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.756002Z","iopub.execute_input":"2022-07-21T11:43:34.756806Z","iopub.status.idle":"2022-07-21T11:43:34.769357Z","shell.execute_reply.started":"2022-07-21T11:43:34.756773Z","shell.execute_reply":"2022-07-21T11:43:34.768114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtFinType1 = df_train.BsmtFinType1.fillna(df_train.BsmtFinType1.mode()[0])\ndf_train.BsmtFinType1.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.771299Z","iopub.execute_input":"2022-07-21T11:43:34.771877Z","iopub.status.idle":"2022-07-21T11:43:34.782797Z","shell.execute_reply.started":"2022-07-21T11:43:34.771835Z","shell.execute_reply":"2022-07-21T11:43:34.781369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('BsmtFinType1', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.794225Z","iopub.execute_input":"2022-07-21T11:43:34.794979Z","iopub.status.idle":"2022-07-21T11:43:34.803734Z","shell.execute_reply.started":"2022-07-21T11:43:34.794940Z","shell.execute_reply":"2022-07-21T11:43:34.802373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtCond.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.805219Z","iopub.execute_input":"2022-07-21T11:43:34.806224Z","iopub.status.idle":"2022-07-21T11:43:34.819985Z","shell.execute_reply.started":"2022-07-21T11:43:34.806143Z","shell.execute_reply":"2022-07-21T11:43:34.819001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtCond= df_train.BsmtCond.fillna(df_train.BsmtCond.mode()[0])\ndf_train.BsmtCond.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.821360Z","iopub.execute_input":"2022-07-21T11:43:34.821850Z","iopub.status.idle":"2022-07-21T11:43:34.832322Z","shell.execute_reply.started":"2022-07-21T11:43:34.821820Z","shell.execute_reply":"2022-07-21T11:43:34.831067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('BsmtCond', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.833565Z","iopub.execute_input":"2022-07-21T11:43:34.833877Z","iopub.status.idle":"2022-07-21T11:43:34.844202Z","shell.execute_reply.started":"2022-07-21T11:43:34.833850Z","shell.execute_reply":"2022-07-21T11:43:34.843219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.845371Z","iopub.execute_input":"2022-07-21T11:43:34.846079Z","iopub.status.idle":"2022-07-21T11:43:34.855500Z","shell.execute_reply.started":"2022-07-21T11:43:34.846049Z","shell.execute_reply":"2022-07-21T11:43:34.854565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtQual.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.856937Z","iopub.execute_input":"2022-07-21T11:43:34.857481Z","iopub.status.idle":"2022-07-21T11:43:34.869981Z","shell.execute_reply.started":"2022-07-21T11:43:34.857450Z","shell.execute_reply":"2022-07-21T11:43:34.869213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.BsmtQual = df_train.BsmtQual.fillna(df_train.BsmtQual.mode()[0])\ndf_train.BsmtQual.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.871245Z","iopub.execute_input":"2022-07-21T11:43:34.871898Z","iopub.status.idle":"2022-07-21T11:43:34.886318Z","shell.execute_reply.started":"2022-07-21T11:43:34.871869Z","shell.execute_reply":"2022-07-21T11:43:34.884828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('BsmtQual', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.887883Z","iopub.execute_input":"2022-07-21T11:43:34.888531Z","iopub.status.idle":"2022-07-21T11:43:34.901011Z","shell.execute_reply.started":"2022-07-21T11:43:34.888487Z","shell.execute_reply":"2022-07-21T11:43:34.899665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.MasVnrArea.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.902357Z","iopub.execute_input":"2022-07-21T11:43:34.903144Z","iopub.status.idle":"2022-07-21T11:43:34.917099Z","shell.execute_reply.started":"2022-07-21T11:43:34.903111Z","shell.execute_reply":"2022-07-21T11:43:34.916217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.MasVnrArea = df_train.MasVnrArea.fillna(df_train.MasVnrArea.median())\ndf_train.MasVnrArea.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.918352Z","iopub.execute_input":"2022-07-21T11:43:34.918945Z","iopub.status.idle":"2022-07-21T11:43:34.926899Z","shell.execute_reply.started":"2022-07-21T11:43:34.918916Z","shell.execute_reply":"2022-07-21T11:43:34.925714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('MasVnrArea', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.928487Z","iopub.execute_input":"2022-07-21T11:43:34.929301Z","iopub.status.idle":"2022-07-21T11:43:34.943607Z","shell.execute_reply.started":"2022-07-21T11:43:34.929258Z","shell.execute_reply":"2022-07-21T11:43:34.942557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.MasVnrType.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.946207Z","iopub.execute_input":"2022-07-21T11:43:34.946696Z","iopub.status.idle":"2022-07-21T11:43:34.955412Z","shell.execute_reply.started":"2022-07-21T11:43:34.946654Z","shell.execute_reply":"2022-07-21T11:43:34.954316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.MasVnrType = df_train.MasVnrType.fillna(df_train.MasVnrType.mode()[0])\ndf_train.MasVnrType.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.957225Z","iopub.execute_input":"2022-07-21T11:43:34.958242Z","iopub.status.idle":"2022-07-21T11:43:34.968771Z","shell.execute_reply.started":"2022-07-21T11:43:34.958198Z","shell.execute_reply":"2022-07-21T11:43:34.967720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing.drop('MasVnrType', axis=0, inplace=True)\nmissing","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.970539Z","iopub.execute_input":"2022-07-21T11:43:34.970983Z","iopub.status.idle":"2022-07-21T11:43:34.980660Z","shell.execute_reply.started":"2022-07-21T11:43:34.970944Z","shell.execute_reply":"2022-07-21T11:43:34.979403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.Electrical.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:34.982394Z","iopub.execute_input":"2022-07-21T11:43:34.983577Z","iopub.status.idle":"2022-07-21T11:43:34.999738Z","shell.execute_reply.started":"2022-07-21T11:43:34.983534Z","shell.execute_reply":"2022-07-21T11:43:34.998436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.Electrical= df_train.Electrical.fillna(df_train.Electrical.mode()[0])\ndf_train.Electrical.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:35.001337Z","iopub.execute_input":"2022-07-21T11:43:35.002475Z","iopub.status.idle":"2022-07-21T11:43:35.011969Z","shell.execute_reply.started":"2022-07-21T11:43:35.002432Z","shell.execute_reply":"2022-07-21T11:43:35.011043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:35.013197Z","iopub.execute_input":"2022-07-21T11:43:35.013855Z","iopub.status.idle":"2022-07-21T11:43:35.031746Z","shell.execute_reply.started":"2022-07-21T11:43:35.013825Z","shell.execute_reply":"2022-07-21T11:43:35.030743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Missing values are handled.\nNow checking for datatypes","metadata":{}},{"cell_type":"code","source":"num_cols= list(df_train.select_dtypes(['int64', 'float64']).columns)\nprint(num_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:35.033004Z","iopub.execute_input":"2022-07-21T11:43:35.034140Z","iopub.status.idle":"2022-07-21T11:43:35.042909Z","shell.execute_reply.started":"2022-07-21T11:43:35.034109Z","shell.execute_reply":"2022-07-21T11:43:35.041520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_to_cat=[]\nfor c in num_cols:\n if len(df_train[c].unique()) < 15:\n num_to_cat.append(c)\n sns.distplot(df_train[c])\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:35.044552Z","iopub.execute_input":"2022-07-21T11:43:35.045453Z","iopub.status.idle":"2022-07-21T11:43:37.547377Z","shell.execute_reply.started":"2022-07-21T11:43:35.045408Z","shell.execute_reply":"2022-07-21T11:43:37.546476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in num_to_cat:\n df_train[c]= df_train[c].astype('object')\ndf_train[num_to_cat].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:37.548696Z","iopub.execute_input":"2022-07-21T11:43:37.549474Z","iopub.status.idle":"2022-07-21T11:43:37.571367Z","shell.execute_reply.started":"2022-07-21T11:43:37.549430Z","shell.execute_reply":"2022-07-21T11:43:37.570199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Explaratory Data Analysis","metadata":{}},{"cell_type":"code","source":"num_cols= list(df_train.select_dtypes(['int64', 'float64']).columns)\ncat_cols= list(df_train.select_dtypes('object').columns)\nprint(\"NUMERICAL\")\nprint(num_cols)\nprint()\nprint(\"CATEGORICAL\")\nprint(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:37.572873Z","iopub.execute_input":"2022-07-21T11:43:37.573563Z","iopub.status.idle":"2022-07-21T11:43:37.586540Z","shell.execute_reply.started":"2022-07-21T11:43:37.573522Z","shell.execute_reply":"2022-07-21T11:43:37.585033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Numerical Columns","metadata":{}},{"cell_type":"code","source":"for col in num_cols:\n sns.distplot(df_train[col])\n plt.figtext(1.0, 0.5, s= str(df_train[col].describe()), fontsize=12)\n plt.figtext(1.5, 0.5, s= str(df_train[col].value_counts(normalize=True).sort_values(ascending=False).head()), fontsize=12)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:37.588442Z","iopub.execute_input":"2022-07-21T11:43:37.589183Z","iopub.status.idle":"2022-07-21T11:43:44.841537Z","shell.execute_reply.started":"2022-07-21T11:43:37.589123Z","shell.execute_reply":"2022-07-21T11:43:44.840263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dropping the columns which are highly biased\nbiased_cols= ['Id', 'BsmtFinSF2', 'LowQualFinSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'MiscVal']\ncol_to_drop.extend(biased_cols)\ndf_train.drop(biased_cols, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:44.843392Z","iopub.execute_input":"2022-07-21T11:43:44.844130Z","iopub.status.idle":"2022-07-21T11:43:44.852833Z","shell.execute_reply.started":"2022-07-21T11:43:44.844086Z","shell.execute_reply":"2022-07-21T11:43:44.851606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Categorical Columns","metadata":{}},{"cell_type":"code","source":"for col in cat_cols:\n sns.countplot(df_train[col])\n plt.figtext(1.0, 0.5, s= str(df_train[col].value_counts(normalize=True)), fontsize=14)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:44.854447Z","iopub.execute_input":"2022-07-21T11:43:44.855445Z","iopub.status.idle":"2022-07-21T11:43:54.466741Z","shell.execute_reply.started":"2022-07-21T11:43:44.855412Z","shell.execute_reply":"2022-07-21T11:43:54.465916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.MSZoning= df_train.MSZoning.apply(lambda x: 'RL' if x== 'RL' else 'other')\ndf_train.MSZoning.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.468279Z","iopub.execute_input":"2022-07-21T11:43:54.469345Z","iopub.status.idle":"2022-07-21T11:43:54.479762Z","shell.execute_reply.started":"2022-07-21T11:43:54.469301Z","shell.execute_reply":"2022-07-21T11:43:54.478930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.LotShape= df_train.LotShape.apply(lambda x: x if x == 'Reg' else 'Irregular')\ndf_train.LandContour = df_train.LandContour.apply(lambda x: x if x == 'Lvl' else 'NonLvl')\ndf_train.LotConfig = df_train.LotConfig.apply(lambda x: x if x in ['Inside', 'Corner'] else 'Other')\ndf_train.Condition1 = df_train.Condition1.apply(lambda x: x if x=='Norm' else 'Other')\ndf_train.BldgType = df_train.BldgType.apply(lambda x: x if x=='1Fam' else 'Other')\ndf_train.HouseStyle = df_train.HouseStyle.apply(lambda x: x if x in ['1Story', '2Story'] else 'Other')\ndf_train.RoofStyle= df_train.RoofStyle.apply(lambda x: x if x=='Gable' else 'Non-Gable')\ndf_train.BsmtFinType2 = df_train.BsmtFinType2.apply(lambda x: x if x== 'Unf' else 'Other')\ndf_train.BsmtFullBath = df_train.BsmtFullBath.apply(lambda x: 'No' if x==0 else 'Yes')\ndf_train.SaleType = df_train.SaleType.apply(lambda x: x if x== 'WD' else 'Other')\ndf_train.SaleCondition= df_train.SaleCondition.apply(lambda x: x if x=='Normal' else 'Other') ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.481815Z","iopub.execute_input":"2022-07-21T11:43:54.482263Z","iopub.status.idle":"2022-07-21T11:43:54.501224Z","shell.execute_reply.started":"2022-07-21T11:43:54.482212Z","shell.execute_reply":"2022-07-21T11:43:54.500074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"biased_cat=[]\nfor col in cat_cols:\n val= df_train[col].value_counts(normalize=True).sort_values(ascending=False).iloc[0]\n if val > 0.8:\n biased_cat.append(col)\n \nprint(biased_cat)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.503036Z","iopub.execute_input":"2022-07-21T11:43:54.503879Z","iopub.status.idle":"2022-07-21T11:43:54.558424Z","shell.execute_reply.started":"2022-07-21T11:43:54.503833Z","shell.execute_reply":"2022-07-21T11:43:54.557053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop.extend(biased_cat)\ndf_train.drop(biased_cat, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.560108Z","iopub.execute_input":"2022-07-21T11:43:54.561112Z","iopub.status.idle":"2022-07-21T11:43:54.568205Z","shell.execute_reply.started":"2022-07-21T11:43:54.561066Z","shell.execute_reply":"2022-07-21T11:43:54.566865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bivariate Analysis","metadata":{}},{"cell_type":"code","source":"num_cols= list(df_train.select_dtypes(['int64', 'float64']).columns)\ncat_cols= list(df_train.select_dtypes('object').columns)\nprint(\"Numerical Columns\")\nprint(num_cols)\nprint()\nprint(\"Categorical Columns\")\nprint(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.569835Z","iopub.execute_input":"2022-07-21T11:43:54.570925Z","iopub.status.idle":"2022-07-21T11:43:54.585277Z","shell.execute_reply.started":"2022-07-21T11:43:54.570883Z","shell.execute_reply":"2022-07-21T11:43:54.584256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Numerical","metadata":{}},{"cell_type":"code","source":"for col in num_cols:\n df_train.plot.scatter(x= col, y= 'SalePrice')\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:54.586470Z","iopub.execute_input":"2022-07-21T11:43:54.586945Z","iopub.status.idle":"2022-07-21T11:43:58.062795Z","shell.execute_reply.started":"2022-07-21T11:43:54.586916Z","shell.execute_reply":"2022-07-21T11:43:58.062031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(df_train[num_cols])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:43:58.063918Z","iopub.execute_input":"2022-07-21T11:43:58.064389Z","iopub.status.idle":"2022-07-21T11:44:53.763043Z","shell.execute_reply.started":"2022-07-21T11:43:58.064360Z","shell.execute_reply":"2022-07-21T11:44:53.762139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Categorical","metadata":{}},{"cell_type":"code","source":"for col in cat_cols:\n sns.boxplot(x= df_train[col], y= df_train['SalePrice'])\n plt.xticks(rotation=45)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:44:53.764735Z","iopub.execute_input":"2022-07-21T11:44:53.765051Z","iopub.status.idle":"2022-07-21T11:45:00.672541Z","shell.execute_reply.started":"2022-07-21T11:44:53.765024Z","shell.execute_reply":"2022-07-21T11:45:00.671354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation","metadata":{}},{"cell_type":"code","source":"cor_mat= df_train.corr()\nplt.figure(figsize=(15,15))\nsns.heatmap(cor_mat, annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:00.673971Z","iopub.execute_input":"2022-07-21T11:45:00.675000Z","iopub.status.idle":"2022-07-21T11:45:02.027495Z","shell.execute_reply.started":"2022-07-21T11:45:00.674964Z","shell.execute_reply":"2022-07-21T11:45:02.026385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cor_mat","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:02.028888Z","iopub.execute_input":"2022-07-21T11:45:02.029232Z","iopub.status.idle":"2022-07-21T11:45:02.055780Z","shell.execute_reply.started":"2022-07-21T11:45:02.029200Z","shell.execute_reply":"2022-07-21T11:45:02.054671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr0 = df_train.corr()\n# Removing negative sign from correlation\ncorr0 = corr0.abs()\n#using stack to see the details column wise\ncorr0 = corr0.unstack()\ncorr0 = corr0.sort_values(kind='quicksort').dropna()\ncorr0 = corr0[corr0 !=1 ]\ncorr0.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:02.057442Z","iopub.execute_input":"2022-07-21T11:45:02.058110Z","iopub.status.idle":"2022-07-21T11:45:02.081186Z","shell.execute_reply.started":"2022-07-21T11:45:02.058067Z","shell.execute_reply":"2022-07-21T11:45:02.080042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Outlier Analysis","metadata":{}},{"cell_type":"code","source":"for col in num_cols:\n sns.boxplot(df_train[col])\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:02.083852Z","iopub.execute_input":"2022-07-21T11:45:02.084971Z","iopub.status.idle":"2022-07-21T11:45:05.766832Z","shell.execute_reply.started":"2022-07-21T11:45:02.084928Z","shell.execute_reply":"2022-07-21T11:45:05.765508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replacing the outlier (Q3 + 1.5*IQR) with 1.5*IQR\nfor col in num_cols:\n if col not in \"SalePrice\":\n Q1 = df_train[col].quantile(0.01)\n Q3 = df_train[col].quantile(0.99)\n IQR = Q3 - Q1\n out = Q3 + 1.5 * IQR\n df_train.loc[df_train[col] > out, col] = 1.5 * IQR","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.768186Z","iopub.execute_input":"2022-07-21T11:45:05.768641Z","iopub.status.idle":"2022-07-21T11:45:05.811129Z","shell.execute_reply.started":"2022-07-21T11:45:05.768607Z","shell.execute_reply":"2022-07-21T11:45:05.810346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Outliers are handled","metadata":{}},{"cell_type":"markdown","source":"### Data Pre-Processing","metadata":{}},{"cell_type":"code","source":"y= df_train.pop('SalePrice')\nX= df_train.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.812401Z","iopub.execute_input":"2022-07-21T11:45:05.813478Z","iopub.status.idle":"2022-07-21T11:45:05.821845Z","shell.execute_reply.started":"2022-07-21T11:45:05.813429Z","shell.execute_reply":"2022-07-21T11:45:05.820788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.823258Z","iopub.execute_input":"2022-07-21T11:45:05.823684Z","iopub.status.idle":"2022-07-21T11:45:05.870006Z","shell.execute_reply.started":"2022-07-21T11:45:05.823644Z","shell.execute_reply":"2022-07-21T11:45:05.868708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Converting the categorical features into numbers\ncol= list(X.select_dtypes('object').columns)\n\nfrom sklearn.preprocessing import LabelEncoder\nencode= LabelEncoder()\nX_encode= X[col].apply(encode.fit_transform)\nX= pd.concat([X.select_dtypes(['int64', 'float64']), X_encode], axis=1)\nX.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.872544Z","iopub.execute_input":"2022-07-21T11:45:05.873363Z","iopub.status.idle":"2022-07-21T11:45:05.924057Z","shell.execute_reply.started":"2022-07-21T11:45:05.873330Z","shell.execute_reply":"2022-07-21T11:45:05.922875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Checking the target variable","metadata":{}},{"cell_type":"code","source":"y.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.925675Z","iopub.execute_input":"2022-07-21T11:45:05.926348Z","iopub.status.idle":"2022-07-21T11:45:05.937048Z","shell.execute_reply.started":"2022-07-21T11:45:05.926304Z","shell.execute_reply":"2022-07-21T11:45:05.935991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:05.938527Z","iopub.execute_input":"2022-07-21T11:45:05.939066Z","iopub.status.idle":"2022-07-21T11:45:06.204563Z","shell.execute_reply.started":"2022-07-21T11:45:05.939025Z","shell.execute_reply":"2022-07-21T11:45:06.203450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.skew()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.206248Z","iopub.execute_input":"2022-07-21T11:45:06.206934Z","iopub.status.idle":"2022-07-21T11:45:06.215325Z","shell.execute_reply.started":"2022-07-21T11:45:06.206891Z","shell.execute_reply":"2022-07-21T11:45:06.214232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The target variable is highly skewed, so transforming the variable.","metadata":{}},{"cell_type":"code","source":"y_tr= np.log10(y)\nsns.distplot(y_tr)\nplt.figtext(0.2,0.7, s= \"Skewness=\"+str(round(y_tr.skew(), 3)), fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.217392Z","iopub.execute_input":"2022-07-21T11:45:06.217779Z","iopub.status.idle":"2022-07-21T11:45:06.473508Z","shell.execute_reply.started":"2022-07-21T11:45:06.217749Z","shell.execute_reply":"2022-07-21T11:45:06.472418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now the target variable seems to be normal distributed.","metadata":{}},{"cell_type":"markdown","source":"#### Splitting the dataset in 75:25 ratio","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test= train_test_split(X, y_tr, test_size=0.25, random_state=42)\nprint(X_train.shape, y_train.shape)\nprint(X_test.shape, y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.475510Z","iopub.execute_input":"2022-07-21T11:45:06.476241Z","iopub.status.idle":"2022-07-21T11:45:06.486576Z","shell.execute_reply.started":"2022-07-21T11:45:06.476194Z","shell.execute_reply":"2022-07-21T11:45:06.485759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Scaling ","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler= StandardScaler()\ncols= X_train.columns\nX_train_scaled= scaler.fit_transform(X_train)\nX_train_scaled= pd.DataFrame(X_train_scaled, columns=cols)\nX_train=X_train_scaled.copy()\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.487880Z","iopub.execute_input":"2022-07-21T11:45:06.488770Z","iopub.status.idle":"2022-07-21T11:45:06.534664Z","shell.execute_reply.started":"2022-07-21T11:45:06.488728Z","shell.execute_reply":"2022-07-21T11:45:06.533702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_scaled= scaler.transform(X_test)\nX_test_scaled= pd.DataFrame(X_test_scaled, columns=cols)\nX_test= X_test_scaled.copy()\nX_test.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-21T11:45:06.535976Z","iopub.execute_input":"2022-07-21T11:45:06.536298Z","iopub.status.idle":"2022-07-21T11:45:06.578222Z","shell.execute_reply.started":"2022-07-21T11:45:06.536269Z","shell.execute_reply":"2022-07-21T11:45:06.577046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Building","metadata":{}},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.579572Z","iopub.execute_input":"2022-07-21T11:45:06.579856Z","iopub.status.idle":"2022-07-21T11:45:06.586338Z","shell.execute_reply.started":"2022-07-21T11:45:06.579832Z","shell.execute_reply":"2022-07-21T11:45:06.585205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Importing relevant libraries\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso\nfrom sklearn.feature_selection import RFE\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.587981Z","iopub.execute_input":"2022-07-21T11:45:06.588799Z","iopub.status.idle":"2022-07-21T11:45:06.596842Z","shell.execute_reply.started":"2022-07-21T11:45:06.588756Z","shell.execute_reply":"2022-07-21T11:45:06.595736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Statsmodel","metadata":{}},{"cell_type":"code","source":"#Creating functions for building model and checking VIF\ndef build_model(X,y):\n X = sm.add_constant(X) #Adding the constant\n lm = sm.OLS(y,X).fit() # fitting the model\n print(lm.summary()) # model summary\n return lm\n \ndef checkVIF(X):\n vif = pd.DataFrame()\n vif['Features'] = X.columns\n vif['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]\n vif['VIF'] = round(vif['VIF'], 2)\n vif = vif.sort_values(by = \"VIF\", ascending = False)\n return(vif)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.598476Z","iopub.execute_input":"2022-07-21T11:45:06.599151Z","iopub.status.idle":"2022-07-21T11:45:06.613213Z","shell.execute_reply.started":"2022-07-21T11:45:06.599108Z","shell.execute_reply":"2022-07-21T11:45:06.612122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_model(np.array(X_train), np.array(y_train))\nprint(checkVIF(X_train))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:06.614784Z","iopub.execute_input":"2022-07-21T11:45:06.615510Z","iopub.status.idle":"2022-07-21T11:45:07.036325Z","shell.execute_reply.started":"2022-07-21T11:45:06.615467Z","shell.execute_reply":"2022-07-21T11:45:07.035072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There are so many columns with high p-value and also high VIF score. Its difficult to eliminate the features manually, therefore, using Recusrsive Feature Elimination method.","metadata":{}},{"cell_type":"code","source":"lin_model= LinearRegression()\nlin_model.fit(X_train, y_train)\nrfe= RFE(lin_model, n_features_to_select=15)\nrfe=rfe.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.037955Z","iopub.execute_input":"2022-07-21T11:45:07.038663Z","iopub.status.idle":"2022-07-21T11:45:07.190558Z","shell.execute_reply.started":"2022-07-21T11:45:07.038614Z","shell.execute_reply":"2022-07-21T11:45:07.189232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col= list(X_train.columns[rfe.support_])\nbuild_model(np.array(X_train[col]), np.array(y_train))\nprint(checkVIF(X_train[col]))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.192653Z","iopub.execute_input":"2022-07-21T11:45:07.193640Z","iopub.status.idle":"2022-07-21T11:45:07.273187Z","shell.execute_reply.started":"2022-07-21T11:45:07.193587Z","shell.execute_reply":"2022-07-21T11:45:07.271861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'GrLivArea' variable has very high VIF, so eliminating this variable\ncol.remove('GrLivArea')\nprint(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.275433Z","iopub.execute_input":"2022-07-21T11:45:07.276413Z","iopub.status.idle":"2022-07-21T11:45:07.285648Z","shell.execute_reply.started":"2022-07-21T11:45:07.276363Z","shell.execute_reply":"2022-07-21T11:45:07.284307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_model(np.array(X_train[col]), np.array(y_train))\nprint(checkVIF(X_train[col]))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.289092Z","iopub.execute_input":"2022-07-21T11:45:07.290641Z","iopub.status.idle":"2022-07-21T11:45:07.373765Z","shell.execute_reply.started":"2022-07-21T11:45:07.290589Z","shell.execute_reply":"2022-07-21T11:45:07.372474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The model is quite good with adjusted r2 value as 85.9%.\n* All the variables have p-value less than 0.05\n* The VIF score for all variables is less than 5.","metadata":{}},{"cell_type":"markdown","source":"### 2. Linear Regression","metadata":{}},{"cell_type":"code","source":"lr_model= LinearRegression()\nlr_model.fit(X_train[col], y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.375976Z","iopub.execute_input":"2022-07-21T11:45:07.376944Z","iopub.status.idle":"2022-07-21T11:45:07.416678Z","shell.execute_reply.started":"2022-07-21T11:45:07.376894Z","shell.execute_reply":"2022-07-21T11:45:07.415216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lin_mse_train= round(metrics.mean_squared_error(y_train, lr_model.predict(X_train[col])),3)\nlin_mse_test= round(metrics.mean_squared_error(y_test, lr_model.predict(X_test[col])),3)\n\nprint('mse for train', lin_mse_train)\nprint('mse for test', lin_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.418890Z","iopub.execute_input":"2022-07-21T11:45:07.419363Z","iopub.status.idle":"2022-07-21T11:45:07.468224Z","shell.execute_reply.started":"2022-07-21T11:45:07.419321Z","shell.execute_reply":"2022-07-21T11:45:07.466901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lin_r2_train= round(metrics.r2_score(y_train, lr_model.predict(X_train[col])),3)\nlin_r2_test= round(metrics.r2_score(y_test, lr_model.predict(X_test[col])),3)\n\nprint('r2 for train', lin_r2_train)\nprint('r2 for test', lin_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.469929Z","iopub.execute_input":"2022-07-21T11:45:07.470393Z","iopub.status.idle":"2022-07-21T11:45:07.523693Z","shell.execute_reply.started":"2022-07-21T11:45:07.470353Z","shell.execute_reply":"2022-07-21T11:45:07.522222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Linear Regression Assumption\n1. Assumption 1: There should be a linear relationship between Dependent Variable and Independent Variables\nAs we have already seen above using pair plot, there are some independent variables which shows linear relationship with Target Variable.\n\n2. Assumption 2: Normal distribution of Error terms, i.e. mean of error terms is 0","metadata":{}},{"cell_type":"code","source":"residual= lr_model.predict(X_train[col]) - y_train\nsns.distplot(residual)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.525204Z","iopub.execute_input":"2022-07-21T11:45:07.525719Z","iopub.status.idle":"2022-07-21T11:45:07.835596Z","shell.execute_reply.started":"2022-07-21T11:45:07.525674Z","shell.execute_reply":"2022-07-21T11:45:07.834628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. Assumption 3: Independence of error terms","metadata":{}},{"cell_type":"code","source":"#Plotting a scatter plot\nplt.scatter(residual, lr_model.predict(X_train[col]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:07.836915Z","iopub.execute_input":"2022-07-21T11:45:07.837208Z","iopub.status.idle":"2022-07-21T11:45:08.023623Z","shell.execute_reply.started":"2022-07-21T11:45:07.837182Z","shell.execute_reply":"2022-07-21T11:45:08.022800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Lasso Regularization","metadata":{}},{"cell_type":"code","source":"# list of alphas to tune - if value too high it will lead to underfitting, if it is too low, \n# it will not handle the overfitting\nparams = {'alpha': [0.0001, 0.001,0.002, 0.005,0.01, 0.05, 0.1, \n 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0, \n 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 20, 50, 100, 500, 1000]}\n\nlasso = Lasso()\n\n# cross validation\nfolds = 5\nmodel_cv = GridSearchCV(estimator = lasso, \n param_grid = params, \n scoring= 'neg_mean_squared_error', \n cv = folds, \n return_train_score=True,\n verbose = 1) \nmodel_cv.fit(X_train[col], y_train) ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:08.024766Z","iopub.execute_input":"2022-07-21T11:45:08.025261Z","iopub.status.idle":"2022-07-21T11:45:11.157122Z","shell.execute_reply.started":"2022-07-21T11:45:08.025228Z","shell.execute_reply":"2022-07-21T11:45:11.155714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Plotting the variation of alpha vs negative mean squared error\nplt.figure(figsize=[16,5])\nplt.plot(model_cv.cv_results_['param_alpha'].data, model_cv.cv_results_['mean_train_score'])\nplt.plot(model_cv.cv_results_['param_alpha'].data, model_cv.cv_results_['mean_test_score'])\nplt.xscale(\"log\")\nplt.xlabel('Alpha')\nplt.ylabel('Negative Mean Squared Error')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:11.164395Z","iopub.execute_input":"2022-07-21T11:45:11.168815Z","iopub.status.idle":"2022-07-21T11:45:11.986072Z","shell.execute_reply.started":"2022-07-21T11:45:11.168751Z","shell.execute_reply":"2022-07-21T11:45:11.984897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cv.best_params_['alpha']","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:11.987511Z","iopub.execute_input":"2022-07-21T11:45:11.987961Z","iopub.status.idle":"2022-07-21T11:45:11.995394Z","shell.execute_reply.started":"2022-07-21T11:45:11.987928Z","shell.execute_reply":"2022-07-21T11:45:11.994217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fitting Lasso model for best alpha and printing coefficients which have been penalised\n\nlasso = Lasso(alpha= model_cv.best_params_['alpha'])\n\nlasso.fit(X_train[col], y_train)\nprint(lasso.coef_)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:11.996602Z","iopub.execute_input":"2022-07-21T11:45:11.996996Z","iopub.status.idle":"2022-07-21T11:45:12.012905Z","shell.execute_reply.started":"2022-07-21T11:45:11.996956Z","shell.execute_reply":"2022-07-21T11:45:12.011494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso_mse_train= round(metrics.mean_squared_error(y_train, lasso.predict(X_train[col])),3)\nlasso_mse_test= round(metrics.mean_squared_error(y_test, lasso.predict(X_test[col])),3)\n\nprint('mse for train', lasso_mse_train)\nprint('mse for test', lasso_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.014785Z","iopub.execute_input":"2022-07-21T11:45:12.015546Z","iopub.status.idle":"2022-07-21T11:45:12.041183Z","shell.execute_reply.started":"2022-07-21T11:45:12.015504Z","shell.execute_reply":"2022-07-21T11:45:12.039941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso_r2_train= metrics.r2_score(y_train, lasso.predict(X_train[col]))\nlasso_r2_test= metrics.r2_score(y_test, lasso.predict(X_test[col]))\nprint(lasso_r2_train)\nprint(lasso_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.043523Z","iopub.execute_input":"2022-07-21T11:45:12.045122Z","iopub.status.idle":"2022-07-21T11:45:12.075418Z","shell.execute_reply.started":"2022-07-21T11:45:12.045083Z","shell.execute_reply":"2022-07-21T11:45:12.074284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Random Forest Regressor","metadata":{}},{"cell_type":"code","source":"forest= RandomForestRegressor(random_state=42)\nforest.fit(X_train[col], y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.077730Z","iopub.execute_input":"2022-07-21T11:45:12.079277Z","iopub.status.idle":"2022-07-21T11:45:12.690267Z","shell.execute_reply.started":"2022-07-21T11:45:12.079234Z","shell.execute_reply":"2022-07-21T11:45:12.689071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_mse_train= round(metrics.mean_squared_error(y_train, forest.predict(X_train[col])),3)\nrf_mse_test= round(metrics.mean_squared_error(y_test, forest.predict(X_test[col])),3)\n\nprint('mse for train', rf_mse_train)\nprint('mse for test', rf_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.691713Z","iopub.execute_input":"2022-07-21T11:45:12.692122Z","iopub.status.idle":"2022-07-21T11:45:12.749927Z","shell.execute_reply.started":"2022-07-21T11:45:12.692083Z","shell.execute_reply":"2022-07-21T11:45:12.748760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, forest.predict(X_train[col])))\nprint(metrics.r2_score(y_test, forest.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.753007Z","iopub.execute_input":"2022-07-21T11:45:12.753909Z","iopub.status.idle":"2022-07-21T11:45:12.811458Z","shell.execute_reply.started":"2022-07-21T11:45:12.753872Z","shell.execute_reply":"2022-07-21T11:45:12.810234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Overfitting, Tuning Hyperparameters","metadata":{}},{"cell_type":"code","source":"#n_estimators by varying from its values from 20 to 100 in steps of 20\nparam_test1 = {'n_estimators':np.arange(50,501,20)}\ngsearch1 = GridSearchCV(estimator = RandomForestRegressor(random_state=42), param_grid = param_test1, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngsearch1.fit(X_train[col],y_train)\ngsearch1.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:45:12.813286Z","iopub.execute_input":"2022-07-21T11:45:12.813706Z","iopub.status.idle":"2022-07-21T11:46:14.684588Z","shell.execute_reply.started":"2022-07-21T11:45:12.813665Z","shell.execute_reply":"2022-07-21T11:46:14.683223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf1_mse_train= round(metrics.mean_squared_error(y_train, gsearch1.predict(X_train[col])),3)\nrf1_mse_test= round(metrics.mean_squared_error(y_test, gsearch1.predict(X_test[col])),3)\n\nprint('mse for train', rf1_mse_train)\nprint('mse for test', rf1_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:46:14.686454Z","iopub.execute_input":"2022-07-21T11:46:14.686911Z","iopub.status.idle":"2022-07-21T11:46:14.781609Z","shell.execute_reply.started":"2022-07-21T11:46:14.686868Z","shell.execute_reply":"2022-07-21T11:46:14.780375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, gsearch1.predict(X_train[col])))\nprint(metrics.r2_score(y_test, gsearch1.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:46:14.783965Z","iopub.execute_input":"2022-07-21T11:46:14.784313Z","iopub.status.idle":"2022-07-21T11:46:14.879831Z","shell.execute_reply.started":"2022-07-21T11:46:14.784282Z","shell.execute_reply":"2022-07-21T11:46:14.878635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Tuning max depth and min sample split\n#max_depth by varying from its values from 5 to 20 in steps of 1\nparam_test2 = {'max_depth':np.arange(5,20,1),\n 'min_samples_split': np.arange(10, 101, 20)}\ngsearch2 = GridSearchCV(estimator = RandomForestRegressor(random_state=42, n_estimators=gsearch1.best_params_['n_estimators']), param_grid = param_test2, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngsearch2.fit(X_train[col],y_train)\ngsearch2.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:46:14.881281Z","iopub.execute_input":"2022-07-21T11:46:14.882236Z","iopub.status.idle":"2022-07-21T11:47:32.017960Z","shell.execute_reply.started":"2022-07-21T11:46:14.882193Z","shell.execute_reply":"2022-07-21T11:47:32.016668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf2_mse_train= round(metrics.mean_squared_error(y_train, gsearch2.predict(X_train[col])),3)\nrf2_mse_test= round(metrics.mean_squared_error(y_test, gsearch2.predict(X_test[col])),3)\n\nprint('mse for train', rf2_mse_train)\nprint('mse for test', rf2_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.019248Z","iopub.execute_input":"2022-07-21T11:47:32.019658Z","iopub.status.idle":"2022-07-21T11:47:32.094144Z","shell.execute_reply.started":"2022-07-21T11:47:32.019627Z","shell.execute_reply":"2022-07-21T11:47:32.093074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf2_r2_train= metrics.r2_score(y_train, gsearch2.predict(X_train[col]))\nrf2_r2_test= metrics.r2_score(y_test, gsearch2.predict(X_test[col]))\nprint(rf2_r2_train)\nprint(rf2_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.095806Z","iopub.execute_input":"2022-07-21T11:47:32.096591Z","iopub.status.idle":"2022-07-21T11:47:32.178467Z","shell.execute_reply.started":"2022-07-21T11:47:32.096548Z","shell.execute_reply":"2022-07-21T11:47:32.177180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gradient Boosting","metadata":{}},{"cell_type":"code","source":"gbm= GradientBoostingRegressor(random_state=42)\ngbm.fit(X_train[col], y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.180581Z","iopub.execute_input":"2022-07-21T11:47:32.181024Z","iopub.status.idle":"2022-07-21T11:47:32.402428Z","shell.execute_reply.started":"2022-07-21T11:47:32.180982Z","shell.execute_reply":"2022-07-21T11:47:32.401239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm_mse_train= round(metrics.mean_squared_error(y_train, gbm.predict(X_train[col])),3)\ngbm_mse_test= round(metrics.mean_squared_error(y_test, gbm.predict(X_test[col])),3)\n\nprint('mse for train', gbm_mse_train)\nprint('mse for test', gbm_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.403753Z","iopub.execute_input":"2022-07-21T11:47:32.404133Z","iopub.status.idle":"2022-07-21T11:47:32.421665Z","shell.execute_reply.started":"2022-07-21T11:47:32.404099Z","shell.execute_reply":"2022-07-21T11:47:32.420286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, gbm.predict(X_train[col])))\nprint(metrics.r2_score(y_test, gbm.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.424332Z","iopub.execute_input":"2022-07-21T11:47:32.425059Z","iopub.status.idle":"2022-07-21T11:47:32.444914Z","shell.execute_reply.started":"2022-07-21T11:47:32.424986Z","shell.execute_reply":"2022-07-21T11:47:32.443434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Tuning Hyperparameters","metadata":{}},{"cell_type":"code","source":"#Trying some hyperparameter tuning\nparam_gbm= {'n_estimators': np.arange(100,1001, 50)}\ngbm0= GridSearchCV(estimator=GradientBoostingRegressor(random_state=42),\n param_grid=param_gbm, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngbm0.fit(X_train[col], y_train)\ngbm0.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:47:32.446084Z","iopub.execute_input":"2022-07-21T11:47:32.446780Z","iopub.status.idle":"2022-07-21T11:48:04.169472Z","shell.execute_reply.started":"2022-07-21T11:47:32.446738Z","shell.execute_reply":"2022-07-21T11:48:04.168400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm0_mse_train= round(metrics.mean_squared_error(y_train, gbm0.predict(X_train[col])),3)\ngbm0_mse_test= round(metrics.mean_squared_error(y_test, gbm0.predict(X_test[col])),3)\n\nprint('mse for train', gbm0_mse_train)\nprint('mse for test', gbm0_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:04.170920Z","iopub.execute_input":"2022-07-21T11:48:04.171521Z","iopub.status.idle":"2022-07-21T11:48:04.187891Z","shell.execute_reply.started":"2022-07-21T11:48:04.171492Z","shell.execute_reply":"2022-07-21T11:48:04.186757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, gbm0.predict(X_train[col])))\nprint(metrics.r2_score(y_test, gbm0.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:04.189363Z","iopub.execute_input":"2022-07-21T11:48:04.190324Z","iopub.status.idle":"2022-07-21T11:48:04.210682Z","shell.execute_reply.started":"2022-07-21T11:48:04.190280Z","shell.execute_reply":"2022-07-21T11:48:04.209538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Trying some hyperparameter tuning\nparam_gbm= {'max_depth': np.arange(5,20, 2),\n 'min_samples_split': np.arange(20,101,20)}\ngbm1= GridSearchCV(estimator=GradientBoostingRegressor(random_state=42, n_estimators= gbm0.best_params_['n_estimators']),\n param_grid=param_gbm, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngbm1.fit(X_train[col], y_train)\ngbm1.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:04.227697Z","iopub.execute_input":"2022-07-21T11:48:04.228316Z","iopub.status.idle":"2022-07-21T11:48:34.480196Z","shell.execute_reply.started":"2022-07-21T11:48:04.228279Z","shell.execute_reply":"2022-07-21T11:48:34.479082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm1_mse_train= round(metrics.mean_squared_error(y_train, gbm1.predict(X_train[col])),3)\ngbm1_mse_test= round(metrics.mean_squared_error(y_test, gbm1.predict(X_test[col])),3)\n\nprint('mse for train', gbm1_mse_train)\nprint('mse for test', gbm1_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:34.481524Z","iopub.execute_input":"2022-07-21T11:48:34.482619Z","iopub.status.idle":"2022-07-21T11:48:34.500804Z","shell.execute_reply.started":"2022-07-21T11:48:34.482574Z","shell.execute_reply":"2022-07-21T11:48:34.499655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, gbm1.predict(X_train[col])))\nprint(metrics.r2_score(y_test, gbm1.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:34.502411Z","iopub.execute_input":"2022-07-21T11:48:34.502960Z","iopub.status.idle":"2022-07-21T11:48:34.524780Z","shell.execute_reply.started":"2022-07-21T11:48:34.502914Z","shell.execute_reply":"2022-07-21T11:48:34.523936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Trying with different learning rates\n\nparam_gbm= {'learning_rate': [0.005, 0.02, 0.05]}\ngbm2= GridSearchCV(estimator=GradientBoostingRegressor(random_state=42, n_estimators= gbm0.best_params_['n_estimators'],\n max_depth= gbm1.best_params_['max_depth'],\n min_samples_split= gbm1.best_params_['min_samples_split']\n ),\n param_grid=param_gbm, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngbm2.fit(X_train[col], y_train)\ngbm2.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:34.526399Z","iopub.execute_input":"2022-07-21T11:48:34.526754Z","iopub.status.idle":"2022-07-21T11:48:36.452034Z","shell.execute_reply.started":"2022-07-21T11:48:34.526723Z","shell.execute_reply":"2022-07-21T11:48:36.450900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm2_mse_train= round(metrics.mean_squared_error(y_train, gbm2.predict(X_train[col])),3)\ngbm2_mse_test= round(metrics.mean_squared_error(y_test, gbm2.predict(X_test[col])),3)\n\nprint('mse for train', gbm2_mse_train)\nprint('mse for test', gbm2_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:36.453366Z","iopub.execute_input":"2022-07-21T11:48:36.454484Z","iopub.status.idle":"2022-07-21T11:48:36.472808Z","shell.execute_reply.started":"2022-07-21T11:48:36.454439Z","shell.execute_reply":"2022-07-21T11:48:36.472118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, gbm2.predict(X_train[col])))\nprint(metrics.r2_score(y_test, gbm2.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:36.473811Z","iopub.execute_input":"2022-07-21T11:48:36.474557Z","iopub.status.idle":"2022-07-21T11:48:36.496675Z","shell.execute_reply.started":"2022-07-21T11:48:36.474525Z","shell.execute_reply":"2022-07-21T11:48:36.495626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_gbm = {'subsample':[0.6, 0.7, 0.75, 0.8, 0.85, 0.9]}\ngbm3= GridSearchCV(estimator=GradientBoostingRegressor(random_state=42, n_estimators= gbm0.best_params_['n_estimators'],\n max_depth= gbm1.best_params_['max_depth'],\n min_samples_split= gbm1.best_params_['min_samples_split'],\n learning_rate= gbm2.best_params_['learning_rate']\n ),\n param_grid=param_gbm, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\ngbm3.fit(X_train[col], y_train)\ngbm3.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:36.498210Z","iopub.execute_input":"2022-07-21T11:48:36.499312Z","iopub.status.idle":"2022-07-21T11:48:39.160537Z","shell.execute_reply.started":"2022-07-21T11:48:36.499279Z","shell.execute_reply":"2022-07-21T11:48:39.159341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm3_mse_train= round(metrics.mean_squared_error(y_train, gbm3.predict(X_train[col])),3)\ngbm3_mse_test= round(metrics.mean_squared_error(y_test, gbm3.predict(X_test[col])),3)\n\nprint('mse for train', gbm3_mse_train)\nprint('mse for test', gbm3_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.162107Z","iopub.execute_input":"2022-07-21T11:48:39.162599Z","iopub.status.idle":"2022-07-21T11:48:39.181113Z","shell.execute_reply.started":"2022-07-21T11:48:39.162564Z","shell.execute_reply":"2022-07-21T11:48:39.180298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm3_r2_train= metrics.r2_score(y_train, gbm3.predict(X_train[col]))\ngbm3_r2_test= metrics.r2_score(y_test, gbm3.predict(X_test[col]))\nprint(gbm3_r2_train)\nprint(gbm3_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.182521Z","iopub.execute_input":"2022-07-21T11:48:39.183114Z","iopub.status.idle":"2022-07-21T11:48:39.201495Z","shell.execute_reply.started":"2022-07-21T11:48:39.183082Z","shell.execute_reply":"2022-07-21T11:48:39.200585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is quite good model.","metadata":{}},{"cell_type":"markdown","source":"### AdaBoost","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import AdaBoostRegressor\nAB_reg= AdaBoostRegressor(random_state=42)\nAB_reg.fit(X_train[col], y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.202945Z","iopub.execute_input":"2022-07-21T11:48:39.203274Z","iopub.status.idle":"2022-07-21T11:48:39.342534Z","shell.execute_reply.started":"2022-07-21T11:48:39.203244Z","shell.execute_reply":"2022-07-21T11:48:39.341421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ab_mse_train= round(metrics.mean_squared_error(y_train, AB_reg.predict(X_train[col])),3)\nab_mse_test= round(metrics.mean_squared_error(y_test, AB_reg.predict(X_test[col])),3)\n\nprint('mse for train', ab_mse_train)\nprint('mse for test', ab_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.343693Z","iopub.execute_input":"2022-07-21T11:48:39.344526Z","iopub.status.idle":"2022-07-21T11:48:39.373486Z","shell.execute_reply.started":"2022-07-21T11:48:39.344492Z","shell.execute_reply":"2022-07-21T11:48:39.372214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, AB_reg.predict(X_train[col])))\nprint(metrics.r2_score(y_test, AB_reg.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.374866Z","iopub.execute_input":"2022-07-21T11:48:39.375220Z","iopub.status.idle":"2022-07-21T11:48:39.403048Z","shell.execute_reply.started":"2022-07-21T11:48:39.375153Z","shell.execute_reply":"2022-07-21T11:48:39.401932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Tuning Hyperparameters","metadata":{}},{"cell_type":"code","source":"param_ada = {'n_estimators':np.arange(100,501, 20)}\nada1= GridSearchCV(estimator=AdaBoostRegressor(random_state=42),\n param_grid=param_ada, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\nada1.fit(X_train[col], y_train)\nada1.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:48:39.404897Z","iopub.execute_input":"2022-07-21T11:48:39.405251Z","iopub.status.idle":"2022-07-21T11:49:06.533920Z","shell.execute_reply.started":"2022-07-21T11:48:39.405220Z","shell.execute_reply":"2022-07-21T11:49:06.532795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ada_mse_train= round(metrics.mean_squared_error(y_train, ada1.predict(X_train[col])),3)\nada_mse_test= round(metrics.mean_squared_error(y_test, ada1.predict(X_test[col])),3)\n\nprint('mse for train', ada_mse_train)\nprint('mse for test', ada_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:06.535542Z","iopub.execute_input":"2022-07-21T11:49:06.536130Z","iopub.status.idle":"2022-07-21T11:49:06.590866Z","shell.execute_reply.started":"2022-07-21T11:49:06.536077Z","shell.execute_reply":"2022-07-21T11:49:06.589893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(y_train, ada1.predict(X_train[col])))\nprint(metrics.r2_score(y_test, ada1.predict(X_test[col])))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:06.591886Z","iopub.execute_input":"2022-07-21T11:49:06.592617Z","iopub.status.idle":"2022-07-21T11:49:06.640518Z","shell.execute_reply.started":"2022-07-21T11:49:06.592584Z","shell.execute_reply":"2022-07-21T11:49:06.639356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Tuning hyperparameter- learning rate\nparam_ada = {'learning_rate': [0.001, 0.005, 0.02, 0.05]}\nada2= GridSearchCV(estimator=AdaBoostRegressor(random_state=42, n_estimators=ada1.best_params_['n_estimators']),\n param_grid=param_ada, scoring='neg_mean_squared_error',n_jobs=4, cv=5)\nada2.fit(X_train[col], y_train)\nada2.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:06.641663Z","iopub.execute_input":"2022-07-21T11:49:06.641949Z","iopub.status.idle":"2022-07-21T11:49:09.517035Z","shell.execute_reply.started":"2022-07-21T11:49:06.641924Z","shell.execute_reply":"2022-07-21T11:49:09.515852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ada2_mse_train= round(metrics.mean_squared_error(y_train, ada2.predict(X_train[col])),3)\nada2_mse_test= round(metrics.mean_squared_error(y_test, ada2.predict(X_test[col])),3)\n\nprint('mse for train', ada2_mse_train)\nprint('mse for test', ada2_mse_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:09.518523Z","iopub.execute_input":"2022-07-21T11:49:09.518946Z","iopub.status.idle":"2022-07-21T11:49:09.572175Z","shell.execute_reply.started":"2022-07-21T11:49:09.518904Z","shell.execute_reply":"2022-07-21T11:49:09.570933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ada2_r2_train= metrics.r2_score(y_train, ada2.predict(X_train[col]))\nada2_r2_test= metrics.r2_score(y_test, ada2.predict(X_test[col]))\nprint(ada2_r2_train)\nprint(ada2_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:09.573642Z","iopub.execute_input":"2022-07-21T11:49:09.573981Z","iopub.status.idle":"2022-07-21T11:49:09.621150Z","shell.execute_reply.started":"2022-07-21T11:49:09.573950Z","shell.execute_reply":"2022-07-21T11:49:09.619895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Results","metadata":{}},{"cell_type":"code","source":"result= pd.DataFrame({'Train_MSE':[lin_mse_train, lasso_mse_train, rf2_mse_train, gbm3_mse_train, ada2_mse_train],\n 'Test_MSE':[lin_mse_test, lasso_mse_test, rf2_mse_test, gbm3_mse_test, ada2_mse_test],\n 'Train_R2':[lin_r2_train, lasso_r2_train, rf2_r2_train, gbm3_r2_train, ada2_r2_train],\n 'Test_R2':[lin_r2_test, lasso_r2_test, rf2_r2_test, gbm3_r2_test, ada2_r2_test]})\n\nresult['Models']=['Linear Regression','Lasso', 'RandomForest', 'GBM', 'AdaBoost']\nresult.set_index('Models', inplace=True)\nresult","metadata":{"execution":{"iopub.status.busy":"2022-07-21T11:49:15.873233Z","iopub.execute_input":"2022-07-21T11:49:15.873624Z","iopub.status.idle":"2022-07-21T11:49:15.889111Z","shell.execute_reply.started":"2022-07-21T11:49:15.873595Z","shell.execute_reply":"2022-07-21T11:49:15.888322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gadient Boosting Algorithm performed better than other models.","metadata":{}}]}