{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-27T20:53:55.953078Z","iopub.execute_input":"2022-07-27T20:53:55.953486Z","iopub.status.idle":"2022-07-27T20:53:55.962500Z","shell.execute_reply.started":"2022-07-27T20:53:55.953452Z","shell.execute_reply":"2022-07-27T20:53:55.961007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport scipy.stats \nimport math\nimport sklearn\nimport statsmodels.api as sm\nfrom statsmodels.formula.api import ols\nfrom scipy.stats import spearmanr\nfrom sklearn import linear_model\nfrom sklearn import preprocessing\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom statsmodels.formula.api import ols\nfrom statsmodels.stats.anova import anova_lm\n%matplotlib inline\nsns.set_theme(style = \"whitegrid\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T20:53:56.287536Z","iopub.execute_input":"2022-07-27T20:53:56.287970Z","iopub.status.idle":"2022-07-27T20:53:56.304701Z","shell.execute_reply.started":"2022-07-27T20:53:56.287932Z","shell.execute_reply":"2022-07-27T20:53:56.303325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')\ndf_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')\ndf_train['type'] = 'train'\ndf_test['type'] = 'test'\ndf_all = pd.concat([df_train, df_test], axis = 0, ignore_index = True)\ndf_all.set_index('Id', inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T20:53:56.727089Z","iopub.execute_input":"2022-07-27T20:53:56.727997Z","iopub.status.idle":"2022-07-27T20:53:56.804337Z","shell.execute_reply.started":"2022-07-27T20:53:56.727959Z","shell.execute_reply":"2022-07-27T20:53:56.802910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('data1 dimension: ', df_train.shape)\nprint('data2 dimension: ', df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T20:53:57.045876Z","iopub.execute_input":"2022-07-27T20:53:57.046674Z","iopub.status.idle":"2022-07-27T20:53:57.053533Z","shell.execute_reply.started":"2022-07-27T20:53:57.046623Z","shell.execute_reply":"2022-07-27T20:53:57.052290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T20:53:57.450015Z","iopub.execute_input":"2022-07-27T20:53:57.450750Z","iopub.status.idle":"2022-07-27T20:53:57.478815Z","shell.execute_reply.started":"2022-07-27T20:53:57.450700Z","shell.execute_reply":"2022-07-27T20:53:57.477662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# relace NA values in features Alley, BsmtQual, BsmtCond, BsmtExposure, BsmtFinType1, BsmtFinType2,\n# FirePlaceQu, GarageType, GarageFinish, GarageQual, GarageCond, PoolQC, Fence, MiscFeature \n# in df_all by their meaning from description\n\ndf_all.Alley.fillna('No alley access', inplace = True)\ndf_all.BsmtQual.fillna('No Basement', inplace = True)\ndf_all.BsmtCond.fillna('No Basement', inplace = True)\ndf_all.BsmtExposure.fillna('No Basement', inplace = True)\ndf_all.BsmtFinType1.fillna('No Basement', inplace = True)\ndf_all.BsmtFinType2.fillna('No Basement', inplace = True)\ndf_all.FireplaceQu.fillna('No Fireplace', inplace = True)\ndf_all.GarageType.fillna('No Garage', inplace = True)\ndf_all.GarageFinish.fillna('No Garage', inplace = True)\ndf_all.GarageQual.fillna('No Garage', inplace = True)\ndf_all.GarageCond.fillna('No Garage', inplace = True)\ndf_all.PoolQC.fillna('No Pool', inplace = True)\ndf_all.Fence.fillna('No Fence', inplace = True)\ndf_all.MiscFeature.fillna('None', inplace = True)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:57.808946Z","iopub.execute_input":"2022-07-27T20:53:57.809698Z","iopub.status.idle":"2022-07-27T20:53:57.832901Z","shell.execute_reply.started":"2022-07-27T20:53:57.809635Z","shell.execute_reply":"2022-07-27T20:53:57.831703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Missing values\ncount = df_all.isna().sum()[df_all.isna().sum() > 0]\npct = df_all.isna().sum()[df_all.isna().sum() > 0] / df_all.shape[0] * 100\nmissing_tab = pd.DataFrame({'Count': count, 'Percent': pct})\nprint(missing_tab)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:58.112337Z","iopub.execute_input":"2022-07-27T20:53:58.112841Z","iopub.status.idle":"2022-07-27T20:53:58.204585Z","shell.execute_reply.started":"2022-07-27T20:53:58.112801Z","shell.execute_reply":"2022-07-27T20:53:58.203221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imputing missing values\nprint(df_all.MasVnrType.value_counts(), '\\n') # we input missing MasVnrType with None and missing \n# MasVnrArea with 0df_all.Alley.fillna('No alley access', inplace = True)\n\nprint(df_all.LotFrontage.describe(), '\\n') # we impute with median\n\nprint(df_all.loc[df_all.GarageCars.isna(), ['GarageType', 'GarageArea', 'GarageCars']]) # we impute \n# GarageArea and GarageCars with mean","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:58.417705Z","iopub.execute_input":"2022-07-27T20:53:58.418081Z","iopub.status.idle":"2022-07-27T20:53:58.437517Z","shell.execute_reply.started":"2022-07-27T20:53:58.418051Z","shell.execute_reply":"2022-07-27T20:53:58.435937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imputing missing values with mode\nimputer_columns = ['MSZoning', 'Utilities', 'Exterior1st', 'Exterior2nd', 'BsmtFinType1',\n                  'BsmtFinType2', 'Electrical', 'KitchenQual', 'Functional', 'SaleType']\nimputer = SimpleImputer(strategy = 'most_frequent')\nimputer.fit(df_all[imputer_columns])\ndf_all[imputer_columns] = imputer.transform(df_all[imputer_columns])\n\n# imputing MasVnrType and MasVnrArea\ndf_all.MasVnrType.fillna('None', inplace = True)\ndf_all.MasVnrArea.fillna(0, inplace = True)\n\n# imputing numeric variables with mean\nimputer_columns = ['LotFrontage', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',\n                  'BsmtFullBath', 'BsmtHalfBath', 'GarageCars', 'GarageArea']\n\nimputer = SimpleImputer(strategy = 'mean')\nimputer.fit(df_all[imputer_columns])\ndf_all[imputer_columns] = imputer.transform(df_all[imputer_columns])\n\n# drop GarageYrBlt and MoSold - unimportant\ndf_all.drop('GarageYrBlt', axis = 1, inplace = True)\ndf_all.drop('MoSold', axis = 1, inplace = True)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:58.857560Z","iopub.execute_input":"2022-07-27T20:53:58.858341Z","iopub.status.idle":"2022-07-27T20:53:58.896900Z","shell.execute_reply.started":"2022-07-27T20:53:58.858302Z","shell.execute_reply":"2022-07-27T20:53:58.895211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No missing values left\ncount = df_all.isna().sum()[df_all.isna().sum() > 0]\npct = df_all.isna().sum()[df_all.isna().sum() > 0] / df_all.shape[0] * 100\nmissing_tab = pd.DataFrame({'Count': count, 'Percent': pct})\nprint(missing_tab)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:59.261913Z","iopub.execute_input":"2022-07-27T20:53:59.262534Z","iopub.status.idle":"2022-07-27T20:53:59.336680Z","shell.execute_reply.started":"2022-07-27T20:53:59.262495Z","shell.execute_reply":"2022-07-27T20:53:59.335474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating new variables\ndf_all['Age'] = df_all.YrSold - df_all.YearBuilt\ndf_all['TotalFlrSF'] = df_all['1stFlrSF'] + df_all['2ndFlrSF'] + df_all['TotalBsmtSF']\ndf_all['LowQualPctTotalFlrSF'] = df_all.LowQualFinSF / df_all.TotalFlrSF\ndf_all['SoldNew'] = np.where(df_all.YrSold - df_all.YearBuilt <= 1, '1', '0')\n# df_all['SoldNew'] = df_all['SoldNew'].astype('category')\ndf_all['TotalNnumberOfBathrooms'] = df_all.BsmtFullBath + 0.5 * df_all.BsmtHalfBath + df_all.FullBath + 0.5 * df_all.HalfBath\n\ndf_all.drop('1stFlrSF', axis = 1, inplace = True)   \ndf_all.drop('2ndFlrSF', axis = 1, inplace = True)   \ndf_all.drop('TotalBsmtSF', axis = 1, inplace = True)   \ndf_all.drop('LowQualFinSF', axis = 1, inplace = True)   \ndf_all.drop('YrSold', axis = 1, inplace = True)   \ndf_all.drop('YearBuilt', axis = 1, inplace = True)   \ndf_all.drop('BsmtFullBath', axis = 1, inplace = True)   \ndf_all.drop('BsmtHalfBath', axis = 1, inplace = True)   \ndf_all.drop('FullBath', axis = 1, inplace = True)   \ndf_all.drop('HalfBath', axis = 1, inplace = True)   ","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:53:59.676785Z","iopub.execute_input":"2022-07-27T20:53:59.677492Z","iopub.status.idle":"2022-07-27T20:53:59.719136Z","shell.execute_reply.started":"2022-07-27T20:53:59.677438Z","shell.execute_reply":"2022-07-27T20:53:59.717866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make list of numeric and categorical columns\nnumeric = [i for i in df_all.columns if df_all.dtypes[i] != 'object']\ncategorical = [i for i in df_all.columns if df_all.dtypes[i] == 'object']\ncategorical.remove('type')\n\n# convert feature MSSubClass to category\ndf_all['MSSubClass'] = df_all['MSSubClass'].apply(str).astype('category')\nnumeric.remove('MSSubClass')\ncategorical.append('MSSubClass')\n\nprint(f'{len(numeric)} numeric features: \\n{numeric}\\n')\nprint(f'{len(categorical)} categorical features: \\n{categorical}\\n')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:00.514468Z","iopub.execute_input":"2022-07-27T20:54:00.514858Z","iopub.status.idle":"2022-07-27T20:54:00.538508Z","shell.execute_reply.started":"2022-07-27T20:54:00.514827Z","shell.execute_reply":"2022-07-27T20:54:00.537212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numeric variables distributions\ndf = pd.melt(df_all, value_vars = [x for x in numeric if x != 'SalePrice'])\ng = sns.FacetGrid(df, col = \"variable\", col_wrap = 4, sharex = False, sharey = False)\ng = g.map(sns.histplot, \"value\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:01.026814Z","iopub.execute_input":"2022-07-27T20:54:01.027925Z","iopub.status.idle":"2022-07-27T20:54:10.933058Z","shell.execute_reply.started":"2022-07-27T20:54:01.027870Z","shell.execute_reply":"2022-07-27T20:54:10.931947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SalePrice is not normally distributed\nax = sns.histplot(df_all.SalePrice, bins = 40, stat = 'density')\nmu, std = scipy.stats.norm.fit(df_all.SalePrice.dropna())\nxx = np.linspace(*ax.get_xlim(), 100)\nax.plot(xx, scipy.stats.norm.pdf(xx, mu, std), color = 'darkblue')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:10.935458Z","iopub.execute_input":"2022-07-27T20:54:10.936174Z","iopub.status.idle":"2022-07-27T20:54:11.283005Z","shell.execute_reply.started":"2022-07-27T20:54:10.936126Z","shell.execute_reply":"2022-07-27T20:54:11.282104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# log(SalePrice) distribution\nlog_SalePrice = np.log(df_all.SalePrice.dropna())\nax = sns.histplot(log_SalePrice, bins = 40, stat = 'density')\nmu, std = scipy.stats.norm.fit(log_SalePrice.dropna())\nxx = np.linspace(*ax.get_xlim(), 100)\nax.plot(xx, scipy.stats.norm.pdf(xx, mu, std), color = 'darkblue')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:11.284318Z","iopub.execute_input":"2022-07-27T20:54:11.284677Z","iopub.status.idle":"2022-07-27T20:54:11.626812Z","shell.execute_reply.started":"2022-07-27T20:54:11.284644Z","shell.execute_reply":"2022-07-27T20:54:11.625533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numeric features vs. SalePrice correlations\n\n# The following two tables show the first 10 absolute value of Pearson and Spearman correlation \n# (to capture possible non-linear relationships) coefficients between numeric features and \n# SalePrice variable sorted from highest to lowest correlation.\n\ndf_all_subset = df_all[~df_all.SalePrice.isna()] # only use rows from training set\n\n# Pearson correlation coeffs\nPearson_corr = pd.DataFrame(df_all_subset[numeric].corr().abs().SalePrice.sort_values(ascending = False)[1:])\nPearson_corr['Feature'] = Pearson_corr.index\nPearson_corr.reset_index(drop = True, inplace = True)\nPearson_corr.rename(columns={'Feature': \"Feature\", 'SalePrice': \"Pearson\"}, inplace = True)\nPearson_corr = Pearson_corr[['Feature', 'Pearson']]\nprint(Pearson_corr[:10])\n\n# Spearman correlation coeffs\nSpearman_corr = []\nfor i in numeric:\n        if i != 'SalePrice':\n            coef, p = spearmanr(df_all_subset.SalePrice, df_all_subset[i])\n            Spearman_corr.append([i, abs(coef)])\n            \nSpearman_corr = pd.DataFrame(Spearman_corr)\nSpearman_corr.rename(columns = {0: \"Feature\", 1: \"Spearman\"}, inplace = True)\nSpearman_corr.sort_values('Spearman', ascending = False, inplace = True)\nSpearman_corr = Spearman_corr[['Feature', 'Spearman']]\nprint('\\n', Spearman_corr[:10])","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:11.629518Z","iopub.execute_input":"2022-07-27T20:54:11.629902Z","iopub.status.idle":"2022-07-27T20:54:11.693312Z","shell.execute_reply.started":"2022-07-27T20:54:11.629859Z","shell.execute_reply":"2022-07-27T20:54:11.692031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Multicolinearity\n\n# The two following tables show absolute values of Pearson and Spearman correlation \n# coefficients between all pairs of explanatory numeric variables sorted from highest \n# to lowets correlation.\n\n# Pearson\nPearson_corr = df_all[[x for x in numeric if x != 'SalePrice']].corr().abs()\nPearson_corr = Pearson_corr.unstack()\nPearson_corr = Pearson_corr.sort_values(kind = \"quicksort\", ascending = False)\nPearson_corr = Pearson_corr[Pearson_corr != 1].drop_duplicates()\n\nfeature1 = []\nfor pair in list(Pearson_corr.index):\n    feature1.append(pair[0])\n    \nfeature2 = []\nfor pair in list(Pearson_corr.index):\n    feature2.append(pair[1])\n    \nPearson_corr = pd.DataFrame(Pearson_corr)\nPearson_corr.reset_index(drop = True, inplace = True)\nPearson_corr.rename(columns={0: \"Pearson\"}, inplace = True)\nPearson_corr['Feature1'] = feature1\nPearson_corr['Feature2'] = feature2    \nPearson_corr = Pearson_corr[['Feature1', 'Feature2', 'Pearson']]\nprint(Pearson_corr[:10])\n\n# Spearman\nSpearman_corr = []\nfor i in [x for x in numeric if x != 'SalePrice']:\n    for j in [x for x in numeric if x != 'SalePrice']:\n        if i != j:\n            \n            coef, p = spearmanr(df_all[i], df_all[j])\n            #calculate Spearmann correlation coefficient \n            Spearman_corr.append([i, j, abs(coef)])\n            \nSpearman_corr = pd.DataFrame(Spearman_corr)\nSpearman_corr.rename(columns={0: \"Feature1\", 1: \"Feature2\", 2: \"Spearman\"}, inplace = True)\nSpearman_corr.sort_values('Spearman', ascending = False, inplace = True)\nSpearman_corr.reset_index(drop = True, inplace = True)\nSpearman_corr = Spearman_corr.iloc[::2, :]\nprint('\\n', Spearman_corr[:10])","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:11.695009Z","iopub.execute_input":"2022-07-27T20:54:11.695380Z","iopub.status.idle":"2022-07-27T20:54:12.650655Z","shell.execute_reply.started":"2022-07-27T20:54:11.695346Z","shell.execute_reply":"2022-07-27T20:54:12.649344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pearson Chi square test of independence for all pairs of categorical features\n\n# The following table shows the first 20 values of Cramer's V for every pair \n# of categorical features, sorted from highest to lowest.\n\nCramersV = []\nfor i in categorical:\n    for j in categorical:\n        if i != j:\n            ct = pd.crosstab(index = df_all[i], columns = df_all[j])\n            stat, p, dof, expected = scipy.stats.chi2_contingency(ct)\n            n = np.sum(np.sum(ct))\n            minDim = min(ct.shape)-1\n            \n            #calculate Cramer's V \n            V = (np.sqrt((stat/n) / minDim))\n            CramersV.append([i, j, V])\n\n\nCramersV = pd.DataFrame(CramersV)\nCramersV.rename(columns = {0: \"Feature1\", 1: \"Feature2\", 2: \"Cramers_V\"}, inplace = True)\nCramersV.sort_values('Cramers_V', ascending = False, inplace = True)\nCramersV = CramersV.iloc[::2,:]\nCramersV.head(10)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:12.652654Z","iopub.execute_input":"2022-07-27T20:54:12.653109Z","iopub.status.idle":"2022-07-27T20:54:36.492038Z","shell.execute_reply.started":"2022-07-27T20:54:12.653062Z","shell.execute_reply":"2022-07-27T20:54:36.490680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting numeric features vs. SalePrice dependencies\ndef scatterplot(x, y, **kwargs):\n    sns.regplot(x = x, y = y)\n    plt.xticks(rotation = 90)\n\nf = pd.melt(df_all, id_vars = ['SalePrice'], value_vars = numeric)\ng = sns.FacetGrid(f, col = \"variable\",  col_wrap = 3, sharex = False, sharey = True, height = 5)\ng = g.map(scatterplot, \"value\", \"SalePrice\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:36.494229Z","iopub.execute_input":"2022-07-27T20:54:36.494741Z","iopub.status.idle":"2022-07-27T20:54:50.939696Z","shell.execute_reply.started":"2022-07-27T20:54:36.494693Z","shell.execute_reply":"2022-07-27T20:54:50.938412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ANOVA: all categorical features x SalePrice\n\n# Now we perform ANOVA for every categorical feature vs. salePrice, \n# choose features which have significant effect on SalePrice (p value < 0.05) \n# and sort the features by p-values from lowest to highest.\n\ndf = pd.melt(df_all_subset, id_vars = ['SalePrice'], value_vars = categorical)\n\nANOVA_df = []\nfor i in categorical:\n    mdl = ols('SalePrice ~ value', data = df[df.variable == i]).fit()\n    p = sm.stats.anova_lm(mdl, typ = 2)['PR(>F)'][0] \n    ANOVA_df.append([i, p])\n\n\nANOVA_df = pd.DataFrame(ANOVA_df).rename(columns = {0: \"Feature\", 1: \"p_value\"})\nANOVA_df = ANOVA_df[ANOVA_df.p_value < 0.05].sort_values('p_value')\nANOVA_df","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:50.941528Z","iopub.execute_input":"2022-07-27T20:54:50.941945Z","iopub.status.idle":"2022-07-27T20:54:52.737049Z","shell.execute_reply.started":"2022-07-27T20:54:50.941909Z","shell.execute_reply":"2022-07-27T20:54:52.735819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SalePrice vs. significant categorical features boxplots from table above\n\ndef boxplot(x, y, **kwargs):\n    sns.boxplot(x = x, y = y, palette = \"cubehelix\")\n    x=plt.xticks(rotation = 90)\n        \nvariables = list(ANOVA_df.Feature)\nvariables.append('SalePrice')\n\nf = pd.melt(df_all_subset[variables], id_vars = ['SalePrice'], value_vars = list(ANOVA_df.Feature))\ng = sns.FacetGrid(f, col = \"variable\",  col_wrap = 3, sharex = False, sharey = False, height = 5)\ng = g.map(boxplot, \"value\", \"SalePrice\")\n","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:54:52.739131Z","iopub.execute_input":"2022-07-27T20:54:52.740125Z","iopub.status.idle":"2022-07-27T20:55:07.655003Z","shell.execute_reply.started":"2022-07-27T20:54:52.740077Z","shell.execute_reply":"2022-07-27T20:55:07.653854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# standardizing numeric columns\nX = df_all[[x for x in numeric if x != 'SalePrice']]\nstandardizer = StandardScaler()\nX = standardizer.fit_transform(X)\ndf_all[[x for x in numeric if x != 'SalePrice']] = X\ndf_all.head()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:07.659788Z","iopub.execute_input":"2022-07-27T20:55:07.660650Z","iopub.status.idle":"2022-07-27T20:55:07.705004Z","shell.execute_reply.started":"2022-07-27T20:55:07.660588Z","shell.execute_reply":"2022-07-27T20:55:07.703746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one-hot encoding categorical variables\ndf_all = pd.get_dummies(df_all, columns = categorical)\nprint(df_all.columns)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:07.706519Z","iopub.execute_input":"2022-07-27T20:55:07.706892Z","iopub.status.idle":"2022-07-27T20:55:07.768891Z","shell.execute_reply.started":"2022-07-27T20:55:07.706861Z","shell.execute_reply":"2022-07-27T20:55:07.767553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T20:55:07.770650Z","iopub.execute_input":"2022-07-27T20:55:07.771140Z","iopub.status.idle":"2022-07-27T20:55:07.799726Z","shell.execute_reply.started":"2022-07-27T20:55:07.771092Z","shell.execute_reply":"2022-07-27T20:55:07.798500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# divide preprocessed data back to train and test set\ndf_train = df_all.loc[df_all['type'] == 'train', :]\ndf_test = df_all.loc[df_all['type'] == 'test', :]\ndf_train.drop(['type'], axis = 1, inplace = True)\ndf_test.drop(['type'], axis = 1, inplace = True)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:07.801526Z","iopub.execute_input":"2022-07-27T20:55:07.802934Z","iopub.status.idle":"2022-07-27T20:55:07.824999Z","shell.execute_reply.started":"2022-07-27T20:55:07.802882Z","shell.execute_reply":"2022-07-27T20:55:07.823667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df_train['SalePrice']\nX = df_train.loc[:, df_train.columns != 'SalePrice']\n\n# Splitting the train data into train and validation set\ntrain_X, val_X, train_y, val_y = train_test_split(X, y, test_size = 0.30, random_state = 1)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:07.826468Z","iopub.execute_input":"2022-07-27T20:55:07.827336Z","iopub.status.idle":"2022-07-27T20:55:07.839310Z","shell.execute_reply.started":"2022-07-27T20:55:07.827301Z","shell.execute_reply":"2022-07-27T20:55:07.838153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\n\n# Lasso\nfrom sklearn.linear_model import Lasso\nmodels['Lasso'] = Lasso(alpha = 0.1, max_iter = 3000)\n\n\n# Ridge\nfrom sklearn.linear_model import Ridge\nmodels['Ridge'] = Ridge(max_iter = 3000)\n\n# Elastic net\nfrom sklearn.linear_model import ElasticNet\nmodels['ElasticNet'] = ElasticNet(alpha = 0.1)\n\n# Gradient Boost\nfrom sklearn.ensemble import GradientBoostingRegressor\nmodels['GradientBoostingRegressor'] = GradientBoostingRegressor()\n\n# Random Forest\nfrom sklearn.ensemble import RandomForestRegressor\nmodels['RandomForestRegressor'] = RandomForestRegressor()\n\n# XGBoost\nfrom xgboost import XGBRegressor\nmodels['XGBoost'] = XGBRegressor() \n\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score\nMSE = {}\n\nfor key in models.keys():\n    # Fit the classifier model\n    models[key].fit(train_X, train_y)\n    \n    # Prediction on validation data\n    predictions = models[key].predict(val_X)\n    \n    from sklearn.metrics import mean_squared_error\n    MSE[key] = mean_squared_error(predictions, val_y)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:07.840940Z","iopub.execute_input":"2022-07-27T20:55:07.841530Z","iopub.status.idle":"2022-07-27T20:55:13.519640Z","shell.execute_reply.started":"2022-07-27T20:55:07.841497Z","shell.execute_reply":"2022-07-27T20:55:13.518397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_model = pd.DataFrame(index = models.keys(), columns = ['MSE'])\ndf_model['MSE'] = MSE.values()\ndf_model.sort_values('MSE')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:13.524110Z","iopub.execute_input":"2022-07-27T20:55:13.525128Z","iopub.status.idle":"2022-07-27T20:55:13.539986Z","shell.execute_reply.started":"2022-07-27T20:55:13.525080Z","shell.execute_reply":"2022-07-27T20:55:13.538663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# FINAL MODEL - Gradient Boost\n\n# fitting model on the whole training set and prediction on test data\nfinal_model_variables = list(train_X.columns)\ntrain_X = df_train[final_model_variables]\ntrain_y = df_train['SalePrice']\n\ntest_X = df_test[final_model_variables]\n\n# final model\nmodel_final = GradientBoostingRegressor()\nmodel_final.fit(train_X, train_y)\n\n# final prediction on test set\ny_pred_final = model_final.predict(test_X)\n\n# saving results to df for submission\nSaleId = list(df_all[df_all['type'] == 'test'].index)\ndata_tuples = list(zip(SaleId, y_pred_final))\nsubmission_df = pd.DataFrame(data_tuples, columns = ['Id','SalePrice'])\nsubmission_df.to_csv('final_submission.csv', index = False)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-07-27T20:55:13.541668Z","iopub.execute_input":"2022-07-27T20:55:13.542048Z","iopub.status.idle":"2022-07-27T20:55:14.608087Z","shell.execute_reply.started":"2022-07-27T20:55:13.542016Z","shell.execute_reply":"2022-07-27T20:55:14.607168Z"},"trusted":true},"execution_count":null,"outputs":[]}]}