{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T12:51:55.524229Z","iopub.execute_input":"2022-07-22T12:51:55.525162Z","iopub.status.idle":"2022-07-22T12:51:55.562926Z","shell.execute_reply.started":"2022-07-22T12:51:55.525033Z","shell.execute_reply":"2022-07-22T12:51:55.561675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression,LassoCV\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.preprocessing import OrdinalEncoder, PolynomialFeatures, LabelEncoder\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.metrics import mean_absolute_error \nfrom sklearn.metrics import mean_squared_error \nfrom sklearn.metrics import median_absolute_error\nimport math","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:55.565222Z","iopub.execute_input":"2022-07-22T12:51:55.566379Z","iopub.status.idle":"2022-07-22T12:51:57.198778Z","shell.execute_reply.started":"2022-07-22T12:51:55.566319Z","shell.execute_reply":"2022-07-22T12:51:57.197293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.201037Z","iopub.execute_input":"2022-07-22T12:51:57.202073Z","iopub.status.idle":"2022-07-22T12:51:57.209295Z","shell.execute_reply.started":"2022-07-22T12:51:57.202012Z","shell.execute_reply":"2022-07-22T12:51:57.207700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.213369Z","iopub.execute_input":"2022-07-22T12:51:57.214998Z","iopub.status.idle":"2022-07-22T12:51:57.276661Z","shell.execute_reply.started":"2022-07-22T12:51:57.214936Z","shell.execute_reply":"2022-07-22T12:51:57.275324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.279007Z","iopub.execute_input":"2022-07-22T12:51:57.280108Z","iopub.status.idle":"2022-07-22T12:51:57.339368Z","shell.execute_reply.started":"2022-07-22T12:51:57.280049Z","shell.execute_reply":"2022-07-22T12:51:57.338450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.341096Z","iopub.execute_input":"2022-07-22T12:51:57.341679Z","iopub.status.idle":"2022-07-22T12:51:57.348553Z","shell.execute_reply.started":"2022-07-22T12:51:57.341638Z","shell.execute_reply":"2022-07-22T12:51:57.347524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## checking if there is any duplicate data\ndf.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.350086Z","iopub.execute_input":"2022-07-22T12:51:57.351111Z","iopub.status.idle":"2022-07-22T12:51:57.396334Z","shell.execute_reply.started":"2022-07-22T12:51:57.351056Z","shell.execute_reply":"2022-07-22T12:51:57.395018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing Id column - redundant\ndf.drop('Id',axis = 1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.397940Z","iopub.execute_input":"2022-07-22T12:51:57.399081Z","iopub.status.idle":"2022-07-22T12:51:57.409255Z","shell.execute_reply.started":"2022-07-22T12:51:57.399020Z","shell.execute_reply":"2022-07-22T12:51:57.407843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# value counts according to data type\ndf.dtypes.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.410983Z","iopub.execute_input":"2022-07-22T12:51:57.412525Z","iopub.status.idle":"2022-07-22T12:51:57.429292Z","shell.execute_reply.started":"2022-07-22T12:51:57.412459Z","shell.execute_reply":"2022-07-22T12:51:57.427850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns.size","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.434747Z","iopub.execute_input":"2022-07-22T12:51:57.435685Z","iopub.status.idle":"2022-07-22T12:51:57.444395Z","shell.execute_reply.started":"2022-07-22T12:51:57.435611Z","shell.execute_reply":"2022-07-22T12:51:57.442765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking null values in different columns\ndf.isnull().sum()[df.isnull().sum() > 0].sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.446782Z","iopub.execute_input":"2022-07-22T12:51:57.447775Z","iopub.status.idle":"2022-07-22T12:51:57.480373Z","shell.execute_reply.started":"2022-07-22T12:51:57.447715Z","shell.execute_reply":"2022-07-22T12:51:57.479035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 19 columns with null values.","metadata":{}},{"cell_type":"code","source":"# Count of values excluding null\ndf.count()[df.count()<df.shape[0]].sort_values()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.481929Z","iopub.execute_input":"2022-07-22T12:51:57.483273Z","iopub.status.idle":"2022-07-22T12:51:57.514181Z","shell.execute_reply.started":"2022-07-22T12:51:57.483215Z","shell.execute_reply":"2022-07-22T12:51:57.512867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[['MSSubClass', 'MoSold']]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.515874Z","iopub.execute_input":"2022-07-22T12:51:57.516944Z","iopub.status.idle":"2022-07-22T12:51:57.521311Z","shell.execute_reply.started":"2022-07-22T12:51:57.516902Z","shell.execute_reply":"2022-07-22T12:51:57.520354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_to_obj_cols = ['MSSubClass', 'MoSold']\n# df[num_to_obj_cols] = df[num_to_obj_cols].astype(object)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.522598Z","iopub.execute_input":"2022-07-22T12:51:57.523451Z","iopub.status.idle":"2022-07-22T12:51:57.534370Z","shell.execute_reply.started":"2022-07-22T12:51:57.523411Z","shell.execute_reply":"2022-07-22T12:51:57.533037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_cat = df.select_dtypes(object).columns.to_list()\ncols_num = df.select_dtypes(np.number).columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.535703Z","iopub.execute_input":"2022-07-22T12:51:57.536617Z","iopub.status.idle":"2022-07-22T12:51:57.553229Z","shell.execute_reply.started":"2022-07-22T12:51:57.536564Z","shell.execute_reply":"2022-07-22T12:51:57.552176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'No. of Categorical columns - {len(cols_cat)}')\nprint(f'No. of Numerical columns - {len(cols_num)}')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.554597Z","iopub.execute_input":"2022-07-22T12:51:57.555496Z","iopub.status.idle":"2022-07-22T12:51:57.568768Z","shell.execute_reply.started":"2022-07-22T12:51:57.555447Z","shell.execute_reply":"2022-07-22T12:51:57.567251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_num_na = df[cols_num].isnull().sum()[df[cols_num].isnull().sum() > 0]\ncols_num_na","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:51:57.571163Z","iopub.execute_input":"2022-07-22T12:51:57.571743Z","iopub.status.idle":"2022-07-22T12:51:57.591845Z","shell.execute_reply.started":"2022-07-22T12:51:57.571689Z","shell.execute_reply":"2022-07-22T12:51:57.590536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.KitchenAbvGr.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T14:12:37.873951Z","iopub.execute_input":"2022-07-22T14:12:37.875021Z","iopub.status.idle":"2022-07-22T14:12:37.889232Z","shell.execute_reply.started":"2022-07-22T14:12:37.874973Z","shell.execute_reply":"2022-07-22T14:12:37.887697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Handling the missing values","metadata":{}},{"cell_type":"code","source":"missing= df.isnull().sum()[df.isnull().sum() >0].sort_values(ascending=False)\n# calculating in percentage\n(missing/df.shape[0])*100","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:01.972322Z","iopub.execute_input":"2022-07-22T12:52:01.973205Z","iopub.status.idle":"2022-07-22T12:52:02.004983Z","shell.execute_reply.started":"2022-07-22T12:52:01.973164Z","shell.execute_reply":"2022-07-22T12:52:02.003719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_drop= list(missing[(missing/df.shape[0])*100 > 80.0].index)\ncolumns_to_drop","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:05.373020Z","iopub.execute_input":"2022-07-22T12:52:05.373650Z","iopub.status.idle":"2022-07-22T12:52:05.384190Z","shell.execute_reply.started":"2022-07-22T12:52:05.373592Z","shell.execute_reply":"2022-07-22T12:52:05.383266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop(columns_to_drop, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:07.311824Z","iopub.execute_input":"2022-07-22T12:52:07.313010Z","iopub.status.idle":"2022-07-22T12:52:07.322053Z","shell.execute_reply.started":"2022-07-22T12:52:07.312946Z","shell.execute_reply":"2022-07-22T12:52:07.320751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## distribution plot for LotFrontage\n \nsns.displot(df.LotFrontage, kde= True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:17.310614Z","iopub.execute_input":"2022-07-22T12:52:17.311065Z","iopub.status.idle":"2022-07-22T12:52:17.857333Z","shell.execute_reply.started":"2022-07-22T12:52:17.311029Z","shell.execute_reply":"2022-07-22T12:52:17.855653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.LotFrontage.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:31.988186Z","iopub.execute_input":"2022-07-22T12:52:31.989169Z","iopub.status.idle":"2022-07-22T12:52:32.006229Z","shell.execute_reply.started":"2022-07-22T12:52:31.989105Z","shell.execute_reply":"2022-07-22T12:52:32.004860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x = df.LotFrontage)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:33.053957Z","iopub.execute_input":"2022-07-22T12:52:33.054428Z","iopub.status.idle":"2022-07-22T12:52:33.224654Z","shell.execute_reply.started":"2022-07-22T12:52:33.054391Z","shell.execute_reply":"2022-07-22T12:52:33.223415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to the presence of outliers as observed from boxplot, imputing the median values for nulls.","metadata":{}},{"cell_type":"code","source":"df.LotFrontage.fillna(df.LotFrontage.median(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:38.751892Z","iopub.execute_input":"2022-07-22T12:52:38.753161Z","iopub.status.idle":"2022-07-22T12:52:38.760692Z","shell.execute_reply.started":"2022-07-22T12:52:38.753109Z","shell.execute_reply":"2022-07-22T12:52:38.759555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Handling the missing values for Basement features namely BsmtQual,BsmtQual,BsmtCond,BsmtExposure,BsmtFinType1,BsmtFinType2\n","metadata":{}},{"cell_type":"code","source":"## Percentage of missing values\n\nd = df[['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2']]\nd.isnull().sum()/len(d)*100","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:39.495466Z","iopub.execute_input":"2022-07-22T12:52:39.497728Z","iopub.status.idle":"2022-07-22T12:52:39.510981Z","shell.execute_reply.started":"2022-07-22T12:52:39.497670Z","shell.execute_reply":"2022-07-22T12:52:39.509727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_columns = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2']\nfor col in b_columns:\n    df[col].fillna(\"NA\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:41.034746Z","iopub.execute_input":"2022-07-22T12:52:41.035739Z","iopub.status.idle":"2022-07-22T12:52:41.046118Z","shell.execute_reply.started":"2022-07-22T12:52:41.035691Z","shell.execute_reply":"2022-07-22T12:52:41.044556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Handling missing values for GarageType,GarageFinish,GarageQual,GarageCond","metadata":{}},{"cell_type":"code","source":"## Percentage of missing values\nd2=df[['GarageType','GarageYrBlt','GarageFinish','GarageQual','GarageCond']]\nd2.isnull().sum()/len(d2)*100\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:41.589077Z","iopub.execute_input":"2022-07-22T12:52:41.590281Z","iopub.status.idle":"2022-07-22T12:52:41.603841Z","shell.execute_reply.started":"2022-07-22T12:52:41.590226Z","shell.execute_reply":"2022-07-22T12:52:41.602730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replace by NA\ng_columns = ['GarageType','GarageFinish','GarageQual','GarageCond']\nfor col in g_columns:\n    df[col].fillna(\"NA\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:41.778464Z","iopub.execute_input":"2022-07-22T12:52:41.779197Z","iopub.status.idle":"2022-07-22T12:52:41.789231Z","shell.execute_reply.started":"2022-07-22T12:52:41.779148Z","shell.execute_reply":"2022-07-22T12:52:41.787766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(df.GarageYrBlt)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:42.044976Z","iopub.execute_input":"2022-07-22T12:52:42.045889Z","iopub.status.idle":"2022-07-22T12:52:42.319256Z","shell.execute_reply.started":"2022-07-22T12:52:42.045824Z","shell.execute_reply":"2022-07-22T12:52:42.317768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.GarageYrBlt.dtype","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:42.413212Z","iopub.execute_input":"2022-07-22T12:52:42.414128Z","iopub.status.idle":"2022-07-22T12:52:42.421953Z","shell.execute_reply.started":"2022-07-22T12:52:42.414081Z","shell.execute_reply":"2022-07-22T12:52:42.420596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.GarageYrBlt.median()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:42.621647Z","iopub.execute_input":"2022-07-22T12:52:42.622456Z","iopub.status.idle":"2022-07-22T12:52:42.631040Z","shell.execute_reply.started":"2022-07-22T12:52:42.622403Z","shell.execute_reply":"2022-07-22T12:52:42.630119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.GarageYrBlt.fillna(df.GarageYrBlt.median(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:42.849727Z","iopub.execute_input":"2022-07-22T12:52:42.850577Z","iopub.status.idle":"2022-07-22T12:52:42.857212Z","shell.execute_reply.started":"2022-07-22T12:52:42.850531Z","shell.execute_reply":"2022-07-22T12:52:42.856141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replacing missing values for MasVnrType\n\ndf.MasVnrType.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:43.165561Z","iopub.execute_input":"2022-07-22T12:52:43.166322Z","iopub.status.idle":"2022-07-22T12:52:43.176245Z","shell.execute_reply.started":"2022-07-22T12:52:43.166275Z","shell.execute_reply":"2022-07-22T12:52:43.174876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Replacing the missing values with mode\n\ndf.loc[df['MasVnrType'].isnull()==True,'MasVnrType']='None'","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:43.324714Z","iopub.execute_input":"2022-07-22T12:52:43.325888Z","iopub.status.idle":"2022-07-22T12:52:43.332493Z","shell.execute_reply.started":"2022-07-22T12:52:43.325842Z","shell.execute_reply":"2022-07-22T12:52:43.331373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replacing missing values for MasVnrArea\n\nsns.distplot(df.MasVnrArea)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:44.166456Z","iopub.execute_input":"2022-07-22T12:52:44.166943Z","iopub.status.idle":"2022-07-22T12:52:44.479399Z","shell.execute_reply.started":"2022-07-22T12:52:44.166902Z","shell.execute_reply":"2022-07-22T12:52:44.478092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.MasVnrArea.fillna(df.MasVnrArea.median(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:44.481530Z","iopub.execute_input":"2022-07-22T12:52:44.481908Z","iopub.status.idle":"2022-07-22T12:52:44.489653Z","shell.execute_reply.started":"2022-07-22T12:52:44.481876Z","shell.execute_reply":"2022-07-22T12:52:44.488398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling the missing value for Electrical\n\ndf.Electrical.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:44.589987Z","iopub.execute_input":"2022-07-22T12:52:44.590505Z","iopub.status.idle":"2022-07-22T12:52:44.600088Z","shell.execute_reply.started":"2022-07-22T12:52:44.590462Z","shell.execute_reply":"2022-07-22T12:52:44.599199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replacing the null values with mode\ndf.loc[df['Electrical'].isnull()==True,'Electrical']='SBrkr'","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:44.789210Z","iopub.execute_input":"2022-07-22T12:52:44.790180Z","iopub.status.idle":"2022-07-22T12:52:44.797910Z","shell.execute_reply.started":"2022-07-22T12:52:44.790132Z","shell.execute_reply":"2022-07-22T12:52:44.796811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Handling the missing values for FireplaceQu\n\ndf.FireplaceQu.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:44.990760Z","iopub.execute_input":"2022-07-22T12:52:44.991590Z","iopub.status.idle":"2022-07-22T12:52:45.002447Z","shell.execute_reply.started":"2022-07-22T12:52:44.991530Z","shell.execute_reply":"2022-07-22T12:52:45.001316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replacing the missing values with NA\ndf.loc[df['FireplaceQu'].isnull()==True,'FireplaceQu']='NA'","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:46.645074Z","iopub.execute_input":"2022-07-22T12:52:46.646004Z","iopub.status.idle":"2022-07-22T12:52:46.654877Z","shell.execute_reply.started":"2022-07-22T12:52:46.645947Z","shell.execute_reply":"2022-07-22T12:52:46.653453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:47.179015Z","iopub.execute_input":"2022-07-22T12:52:47.179515Z","iopub.status.idle":"2022-07-22T12:52:47.199329Z","shell.execute_reply.started":"2022-07-22T12:52:47.179470Z","shell.execute_reply":"2022-07-22T12:52:47.198116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All null values are handeled","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(25,15))\nsns.heatmap(df.drop('SalePrice',axis=1).corr(),cmap=\"BuPu\",annot=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:52:49.567182Z","iopub.execute_input":"2022-07-22T12:52:49.567696Z","iopub.status.idle":"2022-07-22T12:52:55.104483Z","shell.execute_reply.started":"2022-07-22T12:52:49.567635Z","shell.execute_reply":"2022-07-22T12:52:55.101269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Univariate Analysis","metadata":{}},{"cell_type":"code","source":"# features with object datatype\ndf.select_dtypes(object).columns.size","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:53:11.372806Z","iopub.execute_input":"2022-07-22T12:53:11.373455Z","iopub.status.idle":"2022-07-22T12:53:11.386585Z","shell.execute_reply.started":"2022-07-22T12:53:11.373383Z","shell.execute_reply":"2022-07-22T12:53:11.384927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_cat = df.select_dtypes(object).columns.to_list()\ncols_cat","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:53:14.013700Z","iopub.execute_input":"2022-07-22T12:53:14.014168Z","iopub.status.idle":"2022-07-22T12:53:14.027824Z","shell.execute_reply.started":"2022-07-22T12:53:14.014132Z","shell.execute_reply":"2022-07-22T12:53:14.026229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## countplot for datatype with objects\n\nplt.figure(figsize=(20,180))\nplotnumber=1\nfor c in cols_cat:\n    ax=plt.subplot(20,2,plotnumber)\n    b= sns.countplot(x=df[c],palette='Set2')\n    plt.xticks(rotation=70)\n    plotnumber+=1\n    for bar in b.patches:\n        b.annotate(format(bar.get_height()),\n            (bar.get_x() + bar.get_width() / 2,\n            bar.get_height()), ha='center', va='center',\n            size=10, xytext=(0, 6),textcoords='offset points')\nplt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:53:17.849617Z","iopub.execute_input":"2022-07-22T12:53:17.850861Z","iopub.status.idle":"2022-07-22T12:53:25.708403Z","shell.execute_reply.started":"2022-07-22T12:53:17.850808Z","shell.execute_reply":"2022-07-22T12:53:25.707022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_num = df.select_dtypes(np.number).columns\nfor column in cols_num:\n    plt.figure(figsize=(14, 3))\n    sns.set_theme(style=\"whitegrid\")\n    sns.boxplot(df[column])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:54:33.130810Z","iopub.execute_input":"2022-07-22T12:54:33.131290Z","iopub.status.idle":"2022-07-22T12:54:41.043023Z","shell.execute_reply.started":"2022-07-22T12:54:33.131250Z","shell.execute_reply":"2022-07-22T12:54:41.041524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(df.SalePrice,kde=True)\nplt.xticks(rotation=70)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:57:31.257945Z","iopub.execute_input":"2022-07-22T12:57:31.258456Z","iopub.status.idle":"2022-07-22T12:57:31.805659Z","shell.execute_reply.started":"2022-07-22T12:57:31.258417Z","shell.execute_reply":"2022-07-22T12:57:31.804328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling the skewness\nfrom scipy.stats import skew\n\nnumerical_features = df.dtypes[df.dtypes != 'object'].index\n\n# checking the skewness in all the numerical features\nskewed_features = df[numerical_features].apply(lambda x: skew(x.dropna())).sort_values(ascending = False)\n\n# converting the features into a dataframe\nskewness = pd.DataFrame({'skew':skewed_features})\n\n# checking the head of skewness dataset\nskewness","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:58:32.312973Z","iopub.execute_input":"2022-07-22T12:58:32.313507Z","iopub.status.idle":"2022-07-22T12:58:32.350019Z","shell.execute_reply.started":"2022-07-22T12:58:32.313466Z","shell.execute_reply":"2022-07-22T12:58:32.348616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_features.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:58:41.078749Z","iopub.execute_input":"2022-07-22T12:58:41.079849Z","iopub.status.idle":"2022-07-22T12:58:41.087142Z","shell.execute_reply.started":"2022-07-22T12:58:41.079795Z","shell.execute_reply":"2022-07-22T12:58:41.085977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##applying box-cox transformations\n\nskewness = skewness[abs(skewness > 0.8)]\n\n# printing how many features are to be box-cox transformed\nprint(f\"There are {skewness.shape[0]} skewed numerical features to box cox transform\")\n\n# importing box-cox1p\nfrom scipy.special import boxcox1p\n\n# defining skewed features\nskewed_features = skewness.index\n\nlamda = 0.15\nfor features in skewed_features:\n    df[features] += 1\n    df[features] = boxcox1p(df[features], lamda)\ndf[skewed_features] = np.log1p(df[skewed_features])\nprint('Skewness has been Handled using Box Cox Transformation')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:59:00.199668Z","iopub.execute_input":"2022-07-22T12:59:00.200148Z","iopub.status.idle":"2022-07-22T12:59:00.250732Z","shell.execute_reply.started":"2022-07-22T12:59:00.200107Z","shell.execute_reply":"2022-07-22T12:59:00.249241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting all the categorical feature\n\ndata_object = df.select_dtypes(include = \"object\").columns\nprint (data_object)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:59:13.185435Z","iopub.execute_input":"2022-07-22T12:59:13.185948Z","iopub.status.idle":"2022-07-22T12:59:13.195778Z","shell.execute_reply.started":"2022-07-22T12:59:13.185904Z","shell.execute_reply":"2022-07-22T12:59:13.194635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\nfor features in data_object:\n    df[features] = le.fit_transform(df[features].astype(str))\n\nprint (df.info())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:59:53.252988Z","iopub.execute_input":"2022-07-22T12:59:53.253548Z","iopub.status.idle":"2022-07-22T12:59:53.332532Z","shell.execute_reply.started":"2022-07-22T12:59:53.253492Z","shell.execute_reply":"2022-07-22T12:59:53.331088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df['SalePrice']","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:00:02.090698Z","iopub.execute_input":"2022-07-22T13:00:02.091164Z","iopub.status.idle":"2022-07-22T13:00:02.097285Z","shell.execute_reply.started":"2022-07-22T13:00:02.091125Z","shell.execute_reply":"2022-07-22T13:00:02.095258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(y, kde=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:00:10.268075Z","iopub.execute_input":"2022-07-22T13:00:10.268528Z","iopub.status.idle":"2022-07-22T13:00:10.762838Z","shell.execute_reply.started":"2022-07-22T13:00:10.268491Z","shell.execute_reply":"2022-07-22T13:00:10.761914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Scaling","metadata":{}},{"cell_type":"code","source":"## Scaling the features\n## Spliting the variables\n\nx=df.drop('SalePrice',axis=1) ## all the features\ny=df['SalePrice']  ## target variable","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:02:31.171493Z","iopub.execute_input":"2022-07-22T13:02:31.171941Z","iopub.status.idle":"2022-07-22T13:02:31.181898Z","shell.execute_reply.started":"2022-07-22T13:02:31.171905Z","shell.execute_reply":"2022-07-22T13:02:31.180919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using minmax scaler to scale all the datas\n\nfrom sklearn.preprocessing import MinMaxScaler\nmc=MinMaxScaler()\nscaled_x=mc.fit_transform(x)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:02:40.321238Z","iopub.execute_input":"2022-07-22T13:02:40.321707Z","iopub.status.idle":"2022-07-22T13:02:40.336831Z","shell.execute_reply.started":"2022-07-22T13:02:40.321669Z","shell.execute_reply":"2022-07-22T13:02:40.335674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Splitting test and train\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:03:02.333951Z","iopub.execute_input":"2022-07-22T13:03:02.334443Z","iopub.status.idle":"2022-07-22T13:03:02.340326Z","shell.execute_reply.started":"2022-07-22T13:03:02.334402Z","shell.execute_reply":"2022-07-22T13:03:02.339229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train,x_test,y_train,y_test=train_test_split(scaled_x,y,test_size=0.20,random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:16.972765Z","iopub.execute_input":"2022-07-22T13:04:16.973199Z","iopub.status.idle":"2022-07-22T13:04:16.982610Z","shell.execute_reply.started":"2022-07-22T13:04:16.973165Z","shell.execute_reply":"2022-07-22T13:04:16.981284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.shape  ## number of rows and columns  given for training \n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:19.893880Z","iopub.execute_input":"2022-07-22T13:04:19.894335Z","iopub.status.idle":"2022-07-22T13:04:19.902299Z","shell.execute_reply.started":"2022-07-22T13:04:19.894299Z","shell.execute_reply":"2022-07-22T13:04:19.901097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test.shape  ## number of row and columns given for testing\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:21.357938Z","iopub.execute_input":"2022-07-22T13:04:21.358485Z","iopub.status.idle":"2022-07-22T13:04:21.365381Z","shell.execute_reply.started":"2022-07-22T13:04:21.358440Z","shell.execute_reply":"2022-07-22T13:04:21.364542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Linear Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:23.388061Z","iopub.execute_input":"2022-07-22T13:04:23.388588Z","iopub.status.idle":"2022-07-22T13:04:23.394170Z","shell.execute_reply.started":"2022-07-22T13:04:23.388538Z","shell.execute_reply":"2022-07-22T13:04:23.393153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR=LinearRegression()\nLR.fit(x_train,y_train)  ## fitting the training data\n\nx_test_pred_LR=LR.predict(x_test)  ## predicted x test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:24.787906Z","iopub.execute_input":"2022-07-22T13:04:24.788366Z","iopub.status.idle":"2022-07-22T13:04:24.846187Z","shell.execute_reply.started":"2022-07-22T13:04:24.788313Z","shell.execute_reply":"2022-07-22T13:04:24.842791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_pred_LR=LR.predict(x_train) ##predicted x train\n\nx_train_pred_LR","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:26.052252Z","iopub.execute_input":"2022-07-22T13:04:26.052704Z","iopub.status.idle":"2022-07-22T13:04:26.064982Z","shell.execute_reply.started":"2022-07-22T13:04:26.052668Z","shell.execute_reply":"2022-07-22T13:04:26.063534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Linear Regression trainind score is',LR.score(x_train,y_train))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:27.148757Z","iopub.execute_input":"2022-07-22T13:04:27.149209Z","iopub.status.idle":"2022-07-22T13:04:27.159788Z","shell.execute_reply.started":"2022-07-22T13:04:27.149174Z","shell.execute_reply":"2022-07-22T13:04:27.158356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Linear Regression testing score is',LR.score(x_test,y_test))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:04:28.902317Z","iopub.execute_input":"2022-07-22T13:04:28.902812Z","iopub.status.idle":"2022-07-22T13:04:28.913310Z","shell.execute_reply.started":"2022-07-22T13:04:28.902771Z","shell.execute_reply":"2022-07-22T13:04:28.911829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lasso CV","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Lasso, LassoCV\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:47:08.418235Z","iopub.execute_input":"2022-07-22T13:47:08.418719Z","iopub.status.idle":"2022-07-22T13:47:08.424754Z","shell.execute_reply.started":"2022-07-22T13:47:08.418680Z","shell.execute_reply":"2022-07-22T13:47:08.423428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alphas = [0.1,0.3, 0.5, 0.8, 1]\nlassocv = LassoCV(alphas=alphas, cv=5).fit(x,y)\nprint(lassocv)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:47:10.699468Z","iopub.execute_input":"2022-07-22T13:47:10.699952Z","iopub.status.idle":"2022-07-22T13:47:10.783087Z","shell.execute_reply.started":"2022-07-22T13:47:10.699894Z","shell.execute_reply":"2022-07-22T13:47:10.780835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=lassocv.fit(x_train,y_train)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:48:17.855386Z","iopub.execute_input":"2022-07-22T13:48:17.856051Z","iopub.status.idle":"2022-07-22T13:48:17.911652Z","shell.execute_reply.started":"2022-07-22T13:48:17.855943Z","shell.execute_reply":"2022-07-22T13:48:17.910065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_pred_LCV=model.predict(x_train) ##predicted x train\n\nx_train_pred_LCV","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:49:31.504633Z","iopub.execute_input":"2022-07-22T13:49:31.505072Z","iopub.status.idle":"2022-07-22T13:49:31.515543Z","shell.execute_reply.started":"2022-07-22T13:49:31.505038Z","shell.execute_reply":"2022-07-22T13:49:31.514333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test_pred_LCV=model.predict(x_test)  ## predicted x test","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:50:26.999465Z","iopub.execute_input":"2022-07-22T13:50:26.999902Z","iopub.status.idle":"2022-07-22T13:50:27.007151Z","shell.execute_reply.started":"2022-07-22T13:50:26.999865Z","shell.execute_reply":"2022-07-22T13:50:27.005428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.score(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:52:35.899067Z","iopub.execute_input":"2022-07-22T13:52:35.899590Z","iopub.status.idle":"2022-07-22T13:52:35.920073Z","shell.execute_reply.started":"2022-07-22T13:52:35.899538Z","shell.execute_reply":"2022-07-22T13:52:35.918085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mse = mean_squared_error(y_test,x_test_pred_LCV)\nmse","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:55:45.288660Z","iopub.execute_input":"2022-07-22T13:55:45.289120Z","iopub.status.idle":"2022-07-22T13:55:45.298222Z","shell.execute_reply.started":"2022-07-22T13:55:45.289083Z","shell.execute_reply":"2022-07-22T13:55:45.297398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:59:56.792751Z","iopub.execute_input":"2022-07-22T13:59:56.793317Z","iopub.status.idle":"2022-07-22T13:59:56.800567Z","shell.execute_reply.started":"2022-07-22T13:59:56.793269Z","shell.execute_reply":"2022-07-22T13:59:56.799271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso_r2_train= metrics.r2_score(y_train, model.predict(x_train))\nlasso_r2_test= metrics.r2_score(y_test, x_test_pred_LCV)\nprint(lasso_r2_train)\nprint(lasso_r2_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:59:58.004259Z","iopub.execute_input":"2022-07-22T13:59:58.004727Z","iopub.status.idle":"2022-07-22T13:59:58.020605Z","shell.execute_reply.started":"2022-07-22T13:59:58.004690Z","shell.execute_reply":"2022-07-22T13:59:58.019050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}