{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initial data exploration","metadata":{}},{"cell_type":"markdown","source":"Load in packages.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns #for graphs\nimport matplotlib.pyplot as plt  #for graphs\nfrom scipy.stats import chi2_contingency  #for performing chi squared test\nfrom sklearn.feature_selection import VarianceThreshold #to split variables into differnet groups\nfrom sklearn import preprocessing #for variale encoding\nimport copy #for copying variables \nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier #for random forest models\nfrom sklearn.metrics import mean_absolute_error #for evaluating model performance\nfrom sklearn.linear_model import LinearRegression #for linear regression\nfrom sklearn.model_selection import cross_val_score #for cross validating model performance\nfrom xgboost import XGBRegressor #for XGBoost regression model\nfrom scipy.stats import skew\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-30T19:01:28.329615Z","iopub.execute_input":"2022-07-30T19:01:28.330094Z","iopub.status.idle":"2022-07-30T19:01:28.341587Z","shell.execute_reply.started":"2022-07-30T19:01:28.330031Z","shell.execute_reply":"2022-07-30T19:01:28.340741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load in data","metadata":{}},{"cell_type":"code","source":"#description = pd.read_txt(\"../input/house-prices-advanced-regression-techniques/data_description.txt\")\ntest = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/test.csv\")\ntrain = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/train.csv\")\nsample = pd.read_csv(\"../input/house-prices-advanced-regression-techniques/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-30T19:04:04.099189Z","iopub.execute_input":"2022-07-30T19:04:04.099808Z","iopub.status.idle":"2022-07-30T19:04:04.149954Z","shell.execute_reply.started":"2022-07-30T19:04:04.099771Z","shell.execute_reply":"2022-07-30T19:04:04.149240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"View each dataframe","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.123607Z","iopub.execute_input":"2022-07-30T18:10:58.124114Z","iopub.status.idle":"2022-07-30T18:10:58.151296Z","shell.execute_reply.started":"2022-07-30T18:10:58.124066Z","shell.execute_reply":"2022-07-30T18:10:58.150650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.152948Z","iopub.execute_input":"2022-07-30T18:10:58.153315Z","iopub.status.idle":"2022-07-30T18:10:58.181742Z","shell.execute_reply.started":"2022-07-30T18:10:58.153285Z","shell.execute_reply":"2022-07-30T18:10:58.180811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.183608Z","iopub.execute_input":"2022-07-30T18:10:58.183945Z","iopub.status.idle":"2022-07-30T18:10:58.198155Z","shell.execute_reply.started":"2022-07-30T18:10:58.183900Z","shell.execute_reply":"2022-07-30T18:10:58.197501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at distribution of sales price, our target variable. ","metadata":{}},{"cell_type":"code","source":"sns.histplot(\n    train[\"SalePrice\"]\n).set(xlabel = 'Sale Price', ylabel = 'Count')\nplt.axvline(x = train[\"SalePrice\"].mean(), color = \"red\")\nprint(train['SalePrice'].describe())","metadata":{"execution":{"iopub.status.busy":"2022-07-30T19:01:37.734974Z","iopub.execute_input":"2022-07-30T19:01:37.735286Z","iopub.status.idle":"2022-07-30T19:01:38.087680Z","shell.execute_reply.started":"2022-07-30T19:01:37.735250Z","shell.execute_reply":"2022-07-30T19:01:38.086828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sale price seems to have a right skew, with most prices between $100,000-$300,000 and a few high outliers. Since sales price is right skewed, I will normalize it using a log function.","metadata":{}},{"cell_type":"code","source":"train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-30T19:01:41.419819Z","iopub.execute_input":"2022-07-30T19:01:41.420366Z","iopub.status.idle":"2022-07-30T19:01:41.425756Z","shell.execute_reply.started":"2022-07-30T19:01:41.420303Z","shell.execute_reply":"2022-07-30T19:01:41.425150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I will also normalize other numeric variables that are heavily skewed (skew > 0.5)","metadata":{}},{"cell_type":"code","source":"skew_test = train.select_dtypes(include = \"number\").apply(lambda x: skew(x)).sort_values(ascending = False)\nhighly_skewed = skew_test[skew_test > 0.5]\nhighly_skewed_index = highly_skewed.index\ntrain[highly_skewed_index] = train[highly_skewed_index].apply(lambda x: np.log1p(x))\ntest[highly_skewed_index] = test[highly_skewed_index].apply(lambda x:np.log1p(x))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T19:02:44.270941Z","iopub.execute_input":"2022-07-30T19:02:44.271277Z","iopub.status.idle":"2022-07-30T19:02:44.317462Z","shell.execute_reply.started":"2022-07-30T19:02:44.271237Z","shell.execute_reply":"2022-07-30T19:02:44.316608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look to see if any duplicates in training data. ","metadata":{}},{"cell_type":"code","source":"train2 = train.drop([\"Id\"], axis = 1)\ntrain2.duplicated().unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:42:24.252556Z","iopub.execute_input":"2022-07-30T18:42:24.252835Z","iopub.status.idle":"2022-07-30T18:42:24.287752Z","shell.execute_reply.started":"2022-07-30T18:42:24.252804Z","shell.execute_reply":"2022-07-30T18:42:24.286965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No duplicates in training data, so no need to drop any variables.","metadata":{}},{"cell_type":"markdown","source":"# Investigate missing values.","metadata":{}},{"cell_type":"code","source":"missing_variables = train[train.columns[train.isnull().any()]]\nmissing_variables.isnull().sum()\nmissing_variables.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.563441Z","iopub.execute_input":"2022-07-30T18:10:58.563654Z","iopub.status.idle":"2022-07-30T18:10:58.588029Z","shell.execute_reply.started":"2022-07-30T18:10:58.563628Z","shell.execute_reply":"2022-07-30T18:10:58.587019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop variables with over half missing values.","metadata":{}},{"cell_type":"code","source":"train = train.dropna(axis=1, thresh = len(train)/2)\ntrain_cols = [col for col in list(train.columns) if not col == \"SalePrice\"]\ntest = test[train_cols]","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.589426Z","iopub.execute_input":"2022-07-30T18:10:58.589795Z","iopub.status.idle":"2022-07-30T18:10:58.609492Z","shell.execute_reply.started":"2022-07-30T18:10:58.589752Z","shell.execute_reply":"2022-07-30T18:10:58.608642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fill remaining numeric values with median, categorical with mode","metadata":{}},{"cell_type":"code","source":"numeric_columns = train.select_dtypes(include = 'number').columns\nstring_columns = train.select_dtypes(include = 'object').columns\ntrain[numeric_columns] = train[numeric_columns].fillna(train[numeric_columns].median(skipna = True))\ntrain[string_columns] = train[string_columns].fillna(train[string_columns].mode(dropna = True))\n\nnumeric_columns = test.select_dtypes(include = 'number').columns\nstring_columns = test.select_dtypes(include = 'object').columns\ntest[numeric_columns] = test[numeric_columns].fillna(test[numeric_columns].median(skipna = True))\ntest[string_columns] = test[string_columns].fillna(test[string_columns].mode(dropna = True))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.612303Z","iopub.execute_input":"2022-07-30T18:10:58.612698Z","iopub.status.idle":"2022-07-30T18:10:58.717370Z","shell.execute_reply.started":"2022-07-30T18:10:58.612650Z","shell.execute_reply":"2022-07-30T18:10:58.716763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical variable exploration","metadata":{}},{"cell_type":"markdown","source":"Now, we've found the numeric variables we want to include in our model. Let's take a look at the categorical variables to see which ones seem to impact sale price.","metadata":{}},{"cell_type":"code","source":"categorical_data = train.select_dtypes(include = object)\ncategorical_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.718502Z","iopub.execute_input":"2022-07-30T18:10:58.718908Z","iopub.status.idle":"2022-07-30T18:10:58.745202Z","shell.execute_reply.started":"2022-07-30T18:10:58.718857Z","shell.execute_reply":"2022-07-30T18:10:58.744100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I will create a series of count plots to get a sense of the distribution of these categorical variables.","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(round(len(categorical_data.columns)/3),3, figsize = (10, 30))\nfor i, ax in enumerate(fig.axes):\n    if i < len(categorical_data.columns)-1:\n        sns.countplot(x = categorical_data.columns[i], ax = ax, data = categorical_data)\n        ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=90)\nfig.tight_layout()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:10:58.746420Z","iopub.execute_input":"2022-07-30T18:10:58.746665Z","iopub.status.idle":"2022-07-30T18:11:04.481476Z","shell.execute_reply.started":"2022-07-30T18:10:58.746635Z","shell.execute_reply":"2022-07-30T18:11:04.480153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at distribution of sale price within categorical variables. ","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(round(len(categorical_data.columns)/3),3, figsize = (10, 30))\nfor i, ax in enumerate(fig.axes):\n    if i < len(categorical_data.columns)-1:\n        sns.boxplot(x = train[categorical_data.columns[i]], ax = ax, y = train[\"SalePrice\"])\n        ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=90)\nfig.tight_layout()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:04.483054Z","iopub.execute_input":"2022-07-30T18:11:04.483394Z","iopub.status.idle":"2022-07-30T18:11:13.804665Z","shell.execute_reply.started":"2022-07-30T18:11:04.483354Z","shell.execute_reply":"2022-07-30T18:11:13.800858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From looking at the boxplots, there are several variables which do not appear to affect price significantly Before dropping any of these variables, let's convert categorical data to numeric to get a sense of correlation with sale price.","metadata":{}},{"cell_type":"code","source":"label_encoder = preprocessing.LabelEncoder()\ncategorical_data = categorical_data.apply(label_encoder.fit_transform)\ntrain[list(categorical_data.columns)] = train[list(categorical_data.columns)].apply(label_encoder.fit_transform)\ntest[list(categorical_data.columns)] = test[list(categorical_data.columns)].apply(label_encoder.fit_transform)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:13.806408Z","iopub.execute_input":"2022-07-30T18:11:13.807065Z","iopub.status.idle":"2022-07-30T18:11:13.909422Z","shell.execute_reply.started":"2022-07-30T18:11:13.807019Z","shell.execute_reply":"2022-07-30T18:11:13.908282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choosing numeric variables","metadata":{}},{"cell_type":"markdown","source":"There are lots of variables. Let's make this problem more manageable by looking at different types of variables. First, look at numerical values, try to see which ones are correlated with housing prices. If not correlated, then drop variable. Note that ID will not be treated as numeric.","metadata":{}},{"cell_type":"code","source":"numeric_columns = train.select_dtypes(include = np.number)\nnumeric_columns = numeric_columns.drop(\"Id\", axis = 1)\nnumeric_columns","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:13.910946Z","iopub.execute_input":"2022-07-30T18:11:13.911320Z","iopub.status.idle":"2022-07-30T18:11:13.942724Z","shell.execute_reply.started":"2022-07-30T18:11:13.911276Z","shell.execute_reply":"2022-07-30T18:11:13.941884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I will draw a bunch of histograms to get a sense ot the distribution of these numeric variables. ","metadata":{}},{"cell_type":"code","source":"numeric_columns.hist(figsize = (16, 20), bins = 50, color = \"deepskyblue\", edgecolor = \"black\", xlabelsize =8, ylabelsize = 8)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:13.944039Z","iopub.execute_input":"2022-07-30T18:11:13.944310Z","iopub.status.idle":"2022-07-30T18:11:29.772847Z","shell.execute_reply.started":"2022-07-30T18:11:13.944273Z","shell.execute_reply":"2022-07-30T18:11:29.772202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at each variable's correlation with sale price.","metadata":{}},{"cell_type":"code","source":"corr_matrix=numeric_columns.corr()\nprint(corr_matrix[\"SalePrice\"].sort_values(ascending=False))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:29.774166Z","iopub.execute_input":"2022-07-30T18:11:29.774591Z","iopub.status.idle":"2022-07-30T18:11:29.806832Z","shell.execute_reply.started":"2022-07-30T18:11:29.774476Z","shell.execute_reply":"2022-07-30T18:11:29.805824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only select highly correlated features.","metadata":{}},{"cell_type":"code","source":"cut_off = 0.5\ncor_values = abs(corr_matrix[\"SalePrice\"])\nrelevant_numerics = cor_values[cor_values > cut_off]\nrelevant_numerics\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:29.808146Z","iopub.execute_input":"2022-07-30T18:11:29.808385Z","iopub.status.idle":"2022-07-30T18:11:29.817919Z","shell.execute_reply.started":"2022-07-30T18:11:29.808354Z","shell.execute_reply":"2022-07-30T18:11:29.816884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we have values highly correlated with saleprice. Let's filter to these highly correlated variables and create a correlation matrix to check if independent variables are uncorrelated.","metadata":{}},{"cell_type":"code","source":"numeric_columns = numeric_columns[list(relevant_numerics.index)]\ncorr_matrix=numeric_columns.corr()\nprint(corr_matrix[\"SalePrice\"].sort_values(ascending=False))\nsns.heatmap(corr_matrix, annot = True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:29.819422Z","iopub.execute_input":"2022-07-30T18:11:29.819816Z","iopub.status.idle":"2022-07-30T18:11:31.237753Z","shell.execute_reply.started":"2022-07-30T18:11:29.819770Z","shell.execute_reply":"2022-07-30T18:11:31.236640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seems to be a few numeric variables that are highly correlated with one another. OverallQual is highly correlated with SalePrice, which is not an issue since SalePrice is our target variable. \n\nGarageCars and GarageArea have a correlation of 0.88. The garage correlation seems most troubling, and intuitively these variables shouldn't contain much different information. Let's drop garage cars.","metadata":{}},{"cell_type":"code","source":"numeric_columns = numeric_columns.drop(\"GarageCars\", axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:31.239173Z","iopub.execute_input":"2022-07-30T18:11:31.239409Z","iopub.status.idle":"2022-07-30T18:11:31.246497Z","shell.execute_reply.started":"2022-07-30T18:11:31.239378Z","shell.execute_reply":"2022-07-30T18:11:31.245484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling","metadata":{}},{"cell_type":"markdown","source":"Select variables for model, prepare dataframes for input.","metadata":{}},{"cell_type":"code","source":"features = [col for col in list(numeric_columns) if not col == \"SalePrice\"]\nX = train[features]\nX_test = test[features]\ny = train[\"SalePrice\"]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:31.247815Z","iopub.execute_input":"2022-07-30T18:11:31.248096Z","iopub.status.idle":"2022-07-30T18:11:31.262997Z","shell.execute_reply.started":"2022-07-30T18:11:31.248028Z","shell.execute_reply":"2022-07-30T18:11:31.262301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Run linear regression model","metadata":{}},{"cell_type":"code","source":"linear_model = LinearRegression()\nlinear_model.fit(X, y)\npredictions_linear = linear_model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:31.264107Z","iopub.execute_input":"2022-07-30T18:11:31.264981Z","iopub.status.idle":"2022-07-30T18:11:31.289915Z","shell.execute_reply.started":"2022-07-30T18:11:31.264943Z","shell.execute_reply":"2022-07-30T18:11:31.288585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"Look at results","metadata":{}},{"cell_type":"code","source":"r_squared = linear_model.score(X, y)\nprint(f\"R squared: {r_squared}\")\nprint(f\"Coefficients: {linear_model.coef_}\")\nlinear_cv = cross_val_score(linear_model,X, y,cv=5, scoring = \"neg_root_mean_squared_error\")\nprint([-score for score in linear_cv])\nprint(-linear_cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:31.292733Z","iopub.execute_input":"2022-07-30T18:11:31.294434Z","iopub.status.idle":"2022-07-30T18:11:31.403152Z","shell.execute_reply.started":"2022-07-30T18:11:31.294384Z","shell.execute_reply":"2022-07-30T18:11:31.402218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fit XGBoost regressor model.","metadata":{}},{"cell_type":"code","source":"model = XGBRegressor()\nmodel.fit(X, y)\nxgboost_predictions = model.predict(X_test)\n#undo log transformation for final predictions\nxgboost_predictions = np.expm1(xgboost_predictions)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:18:38.965907Z","iopub.execute_input":"2022-07-30T18:18:38.966226Z","iopub.status.idle":"2022-07-30T18:18:39.444226Z","shell.execute_reply.started":"2022-07-30T18:18:38.966196Z","shell.execute_reply":"2022-07-30T18:18:39.443055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at training score, cross validation score for XGBoost model.","metadata":{}},{"cell_type":"code","source":"model.score(X, y) #training score\nxgboost_cv = cross_val_score(model,X, y,cv=5, scoring = \"neg_root_mean_squared_error\")\nprint([-score for score in xgboost_cv])\nprint(-xgboost_cv.mean())\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:21:42.388654Z","iopub.execute_input":"2022-07-30T18:21:42.389412Z","iopub.status.idle":"2022-07-30T18:21:44.332158Z","shell.execute_reply.started":"2022-07-30T18:21:42.389371Z","shell.execute_reply":"2022-07-30T18:21:44.331213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lower root mean squared error than linear regression model, let's use XGBoost for predictions","metadata":{}},{"cell_type":"code","source":"##submission\noutput = pd.DataFrame({'Id': test.Id, 'SalePrice': xgboost_predictions})\noutput.to_csv('submission.csv', index=False)\nprint(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-30T18:11:31.411540Z","iopub.execute_input":"2022-07-30T18:11:31.415365Z","iopub.status.idle":"2022-07-30T18:11:31.444474Z","shell.execute_reply.started":"2022-07-30T18:11:31.415306Z","shell.execute_reply":"2022-07-30T18:11:31.443392Z"},"trusted":true},"execution_count":null,"outputs":[]}]}