{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# **I. Explarotory data analysis**","metadata":{"id":"c9ab7237","execution":{"iopub.status.busy":"2022-07-05T03:32:58.494674Z","iopub.execute_input":"2022-07-05T03:32:58.495793Z","iopub.status.idle":"2022-07-05T03:32:58.501827Z","shell.execute_reply.started":"2022-07-05T03:32:58.495717Z","shell.execute_reply":"2022-07-05T03:32:58.500168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **I.1. General exploration**\n\n\n\n","metadata":{"id":"7c3ddb82"}},{"cell_type":"code","source":"# Load required libraries\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport warnings\nimport statsmodels.api as sm\nimport statsmodels.formula.api as smf\nimport statsmodels.api as sm\nimport sklearn\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.feature_selection import VarianceThreshold\nfrom scipy import stats\nfrom scipy.stats import chi2_contingency\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\n\nwarnings.filterwarnings(\"ignore\")\n%matplotlib inline\nsns.set(rc={\"figure.figsize\": (20, 15)})\nsns.set_style(\"whitegrid\")","metadata":{"id":"4afd9b6f","outputId":"5099bf2b-406a-4e4a-efe5-4b87d244e732","execution":{"iopub.status.busy":"2022-07-05T03:32:58.512947Z","iopub.execute_input":"2022-07-05T03:32:58.513606Z","iopub.status.idle":"2022-07-05T03:32:58.523822Z","shell.execute_reply.started":"2022-07-05T03:32:58.513577Z","shell.execute_reply":"2022-07-05T03:32:58.522776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading Train set\ndf_train = pd.read_csv(\"/kaggle/input/house-prices-advanced-regression-techniques/train.csv\")\nprint(f\"Train set shape:\\n{df_train.shape}\\n\")\n\n# Loading Test set\ndf_test = pd.read_csv(\"/kaggle/input/house-prices-advanced-regression-techniques/test.csv\")\nprint(f\"Test set shape:\\n{df_test.shape}\")","metadata":{"id":"d1833c84","outputId":"524ae19e-f1c5-4d4e-bd84-21415883d8d6","execution":{"iopub.status.busy":"2022-07-05T03:32:58.530909Z","iopub.execute_input":"2022-07-05T03:32:58.531441Z","iopub.status.idle":"2022-07-05T03:32:58.592571Z","shell.execute_reply.started":"2022-07-05T03:32:58.531400Z","shell.execute_reply":"2022-07-05T03:32:58.590906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summary of each of the variables in train set\ndf_train.info()","metadata":{"id":"4ecd935a","scrolled":true,"outputId":"2f2dc9c8-41eb-450f-9d09-25167e960eee","execution":{"iopub.status.busy":"2022-07-05T03:32:58.594843Z","iopub.execute_input":"2022-07-05T03:32:58.595284Z","iopub.status.idle":"2022-07-05T03:32:58.624268Z","shell.execute_reply.started":"2022-07-05T03:32:58.595242Z","shell.execute_reply":"2022-07-05T03:32:58.623173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#summary of dataset\ndf_train.describe()","metadata":{"id":"ICGDMexdlHSX","outputId":"ff5a30fb-b0c2-4b35-c61e-1affb7240815","execution":{"iopub.status.busy":"2022-07-05T03:32:58.627693Z","iopub.execute_input":"2022-07-05T03:32:58.628138Z","iopub.status.idle":"2022-07-05T03:32:58.719533Z","shell.execute_reply.started":"2022-07-05T03:32:58.628104Z","shell.execute_reply":"2022-07-05T03:32:58.718355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can notice that many columns have missing data. I will handlle these in next upcoming cells**","metadata":{"id":"93bdf673"}},{"cell_type":"code","source":"# Checking if all columns are the same in both data set or not using list comprehension function\ndif_1 = [x for x in df_train.columns if x not in df_test.columns]\nprint(f\"Columns present in df_train and absent in df_test: {dif_1}\\n\")\n\ndif_2 = [x for x in df_test.columns if x not in df_train.columns]\nprint(f\"Columns present in df_test set and absent in df_train: {dif_2}\")","metadata":{"id":"4779e4f6","outputId":"27447126-2525-4151-bfbb-92517f52dbdf","execution":{"iopub.status.busy":"2022-07-05T03:32:58.722987Z","iopub.execute_input":"2022-07-05T03:32:58.723403Z","iopub.status.idle":"2022-07-05T03:32:58.731009Z","shell.execute_reply.started":"2022-07-05T03:32:58.723363Z","shell.execute_reply":"2022-07-05T03:32:58.729931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **The 'SalePrice' column present in the train set but not in test set .it is a target variable,we have to predict that variable for test dataset. So no worries**\n* **And remaining columns in both data set are same**","metadata":{"id":"fb8046ea"}},{"cell_type":"code","source":"# Drop the 'Id' column from the train set because this is not usefull for predicton\ndf_train.drop([\"Id\"], axis=1, inplace=True)\n\n# Save the list of 'Id' before dropping it from the test set\nId_test_list = df_test[\"Id\"].tolist()\ndf_test.drop([\"Id\"], axis=1, inplace=True)","metadata":{"id":"761ae926","execution":{"iopub.status.busy":"2022-07-05T03:32:58.732739Z","iopub.execute_input":"2022-07-05T03:32:58.733056Z","iopub.status.idle":"2022-07-05T03:32:58.743665Z","shell.execute_reply.started":"2022-07-05T03:32:58.733026Z","shell.execute_reply":"2022-07-05T03:32:58.742657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **I.2. Numerical features**","metadata":{"id":"4daee1a2"}},{"cell_type":"markdown","source":"### **I.2.1. Explore and clean Numerical features**","metadata":{"id":"b2c0c79b"}},{"cell_type":"code","source":"# Let's select the columns of the train set with numerical data \ndf_train_num = df_train.select_dtypes(include=[\"number\"])\ndf_train_num.shape","metadata":{"id":"02f0313e","outputId":"536a0d31-3786-4697-e074-da98e3bef76a","execution":{"iopub.status.busy":"2022-07-05T03:32:58.744855Z","iopub.execute_input":"2022-07-05T03:32:58.745601Z","iopub.status.idle":"2022-07-05T03:32:58.760414Z","shell.execute_reply.started":"2022-07-05T03:32:58.745569Z","shell.execute_reply":"2022-07-05T03:32:58.759239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's drop quasi-constant features where 95% of the values are similar or constant\nsel = VarianceThreshold(threshold=0.05) # 0.05: drop column where 95% of the values are constant\n\n# fit finds the features with constant variance\nsel.fit(df_train_num.iloc[:, :-1])\n\n# Get the number of features that are not constant\nprint(\"Number of retained features:\",sum(sel.get_support()))\nprint(\"\\nNumber of quasi_constant features:\", {len(df_train_num.iloc[:, :-1].columns) - sum(sel.get_support())})\n\nquasi_constant_features_list = [x for x in df_train_num.iloc[:, :-1].columns \n if x not in df_train_num.iloc[:, :-1].columns[sel.get_support()]]\nprint(\"\\nQuasi-constant features to be dropped:\",quasi_constant_features_list)\n\n# Let's drop these columns from df_train_num \ndf_train_num.drop(quasi_constant_features_list, axis=1, inplace=True)","metadata":{"id":"54bb7f9b","outputId":"c675263e-a306-45bb-cc6f-4c563e999719","execution":{"iopub.status.busy":"2022-07-05T03:32:58.761718Z","iopub.execute_input":"2022-07-05T03:32:58.762163Z","iopub.status.idle":"2022-07-05T03:32:58.780659Z","shell.execute_reply.started":"2022-07-05T03:32:58.762125Z","shell.execute_reply":"2022-07-05T03:32:58.780046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the distribution of all the numerical data\nfig_ = df_train_num.hist(figsize=(25, 24), bins=60, color=\"cyan\",\n edgecolor=\"gray\", xlabelsize=10, ylabelsize=10)","metadata":{"id":"46735b0d","outputId":"ccd3778a-4af8-498b-9942-506e784d4948","execution":{"iopub.status.busy":"2022-07-05T03:32:58.781763Z","iopub.execute_input":"2022-07-05T03:32:58.782121Z","iopub.status.idle":"2022-07-05T03:33:08.629452Z","shell.execute_reply.started":"2022-07-05T03:32:58.782094Z","shell.execute_reply":"2022-07-05T03:33:08.628582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Heatmap for all the remaining numerical data including the taget 'SalePrice'\n# Define the heatmap parameters\npd.options.display.float_format = \"{:,.2f}\".format\n\n# Define correlation matrix\ncorr_matrix = df_train_num.corr()\n\n# Replace correlation < |0.3| by 0 for a better visibility\ncorr_matrix[(corr_matrix < 0.3) & (corr_matrix > -0.3)] = 0\n\n# plot the heatmap\nsns.heatmap(corr_matrix, vmax=1.0, vmin=-1.0, linewidths=0.1,\n annot_kws={\"size\": 9, \"color\": \"black\"},annot=True)","metadata":{"id":"e7bf9d89","outputId":"c809d344-724f-4610-dd84-a58bd030ea1a","execution":{"iopub.status.busy":"2022-07-05T03:33:08.630620Z","iopub.execute_input":"2022-07-05T03:33:08.631107Z","iopub.status.idle":"2022-07-05T03:33:13.567837Z","shell.execute_reply.started":"2022-07-05T03:33:08.631077Z","shell.execute_reply":"2022-07-05T03:33:13.566580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**From the distribution of each numerical variables as well as the heatmap we can notice 18 features that are important and correlated (correlation higher than absolute 0.3) with our target variable 'SalePrice'.**\n \n**We can also notice that a lot of features are correlated with each other. I will handle these correlation while selecting the features for our models.**","metadata":{"id":"a2ef5415"}},{"cell_type":"code","source":"# Let's select features where the correlation with 'SalePrice' is higher than |0.3|\n# -1 because the latest row is SalePrice\ndf_num_corr = df_train_num.corr()[\"SalePrice\"][:-1]\n\n# Correlated features (r2 > 0.5)\nhigh_features_list = df_num_corr[abs(df_num_corr) >= 0.5].sort_values(ascending=False)\nprint(f\"{len(high_features_list)} strongly correlated variables with SalePrice:\\n{high_features_list}\\n\")\n\n# Correlated features (0.3 < r2 < 0.5)\nlow_features_list = df_num_corr[(abs(df_num_corr) < 0.5) & (abs(df_num_corr) >= 0.3)].sort_values(ascending=False)\nprint(f\"{len(low_features_list)} weekly correlated variables with SalePrice:\\n{low_features_list}\")","metadata":{"id":"a4e33586","outputId":"1dfa4b4e-1f62-42ba-b5f7-7c1a97b09d78","execution":{"iopub.status.busy":"2022-07-05T03:33:13.572420Z","iopub.execute_input":"2022-07-05T03:33:13.572762Z","iopub.status.idle":"2022-07-05T03:33:13.587090Z","shell.execute_reply.started":"2022-07-05T03:33:13.572729Z","shell.execute_reply":"2022-07-05T03:33:13.586206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Features with high correlation (higher than 0.5)\nstrong_features = df_num_corr[abs(df_num_corr) >= 0.5].index.tolist()\nstrong_features.append(\"SalePrice\")\n\ndf_strong_features = df_train_num.loc[:, strong_features]\n\nplt.style.use(\"seaborn-whitegrid\") # define figures style\nfig, ax = plt.subplots(round(len(strong_features) / 3), 3)\n\nfor i, ax in enumerate(fig.axes):\n # plot the correlation of each feature with SalePrice\n if i < len(strong_features)-1:\n sns.regplot(x=strong_features[i], y=\"SalePrice\", data=df_strong_features, ax=ax, scatter_kws={\n \"color\": \"cyan\"}, line_kws={\"color\": \"black\"})","metadata":{"id":"0ba88d0f","outputId":"b662258b-35d2-438d-b7ed-9435708386bb","execution":{"iopub.status.busy":"2022-07-05T03:33:13.588426Z","iopub.execute_input":"2022-07-05T03:33:13.588868Z","iopub.status.idle":"2022-07-05T03:33:17.320155Z","shell.execute_reply.started":"2022-07-05T03:33:13.588842Z","shell.execute_reply":"2022-07-05T03:33:17.319245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Above features are strongly/highly correlated variables with regplot**","metadata":{"id":"YXTOsxw4VURy"}},{"cell_type":"code","source":"# Features with low correlation (between 0.3 and 0.5)\nlow_features = df_num_corr[(abs(df_num_corr) >= 0.3) & (\n abs(df_num_corr) < 0.5)].index.tolist()\nlow_features.append(\"SalePrice\")\n\ndf_low_features = df_train_num.loc[:, low_features]\n\nplt.style.use(\"seaborn-whitegrid\") # define figures style\nfig, ax = plt.subplots(round(len(low_features) / 3), 3)\n\nfor i, ax in enumerate(fig.axes):\n # plot the correlation of each feature with SalePrice\n if i < len(low_features) - 1:\n sns.regplot(x=low_features[i], y=\"SalePrice\", data=df_low_features, ax=ax, scatter_kws={\n \"color\": \"deepskyblue\"}, line_kws={\"color\": \"black\"},)","metadata":{"id":"3ee492d3","outputId":"9bdc90bc-88c9-4761-fcaa-a81d9aa52f3e","execution":{"iopub.status.busy":"2022-07-05T03:33:17.321540Z","iopub.execute_input":"2022-07-05T03:33:17.321899Z","iopub.status.idle":"2022-07-05T03:33:20.267364Z","shell.execute_reply.started":"2022-07-05T03:33:17.321853Z","shell.execute_reply":"2022-07-05T03:33:20.266256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Here we can notice several houses where the price is not expensive (less than 250 000 dollars) for the corresponding surface ('TotalBsmtSF', '1stFlrSF', 'GrLivaArea' etc.). It is better to remove these observations to avoid influencing the prediction model.**\n**There is too much price variation for an area equal to 0, e.g. 'WoodDeckSF', 'OpenPorchSF' etc, will drop these outliers at the end of the data mining.**","metadata":{"id":"f2662817"}},{"cell_type":"code","source":"# Define the list of numerical fetaures to keep\nlist_of_numerical_features = strong_features[:-1] + low_features\n\n# Let's select these features form our train set\ndf_train_num = df_train_num.loc[:, list_of_numerical_features]\n\n# The same features are selected from the test set (-1 -> except 'SalePrice')\ndf_test_num = df_test.loc[:, list_of_numerical_features[:-1]]","metadata":{"id":"59a38da5","execution":{"iopub.status.busy":"2022-07-05T03:33:20.268875Z","iopub.execute_input":"2022-07-05T03:33:20.269232Z","iopub.status.idle":"2022-07-05T03:33:20.279359Z","shell.execute_reply.started":"2022-07-05T03:33:20.269175Z","shell.execute_reply":"2022-07-05T03:33:20.278439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Here I kept the 18 most correlated numerical features with 'SalePrice'**","metadata":{"id":"c4359800"}},{"cell_type":"markdown","source":"### **I.2.2. Missing data of Numerical features**","metadata":{"id":"6cf6cb1c"}},{"cell_type":"markdown","source":"### **Train set**","metadata":{"id":"51433bbe"}},{"cell_type":"code","source":"# Check the NaN of the train set by ploting percent of missing values per column\ncolumn_with_nan = df_train_num.columns[df_train_num.isnull().any()]\ncolumn_name = []\npercent_nan = []\n\nfor i in column_with_nan:\n column_name.append(i)\n percent_nan.append(\n round(df_train_num[i].isnull().sum()*100/len(df_train_num), 2))\n\ntab = pd.DataFrame(column_name, columns=[\"Column\"])\ntab[\"Percent_NaN\"] = percent_nan\ntab.sort_values(by=[\"Percent_NaN\"], ascending=False, inplace=True)\n\n\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (10, 7)})\nsns.set_style(\"whitegrid\")\n\n# Plot results\np = sns.barplot(x=\"Percent_NaN\", y=\"Column\", data=tab,\n edgecolor=\"black\", color=\"lightblue\")\n\np.set_title(\"Percent of missing values for column of the train set\\n\", fontsize=20)","metadata":{"id":"78db019b","outputId":"aa3a11e1-be17-4378-d2d6-d6894a85bb07","execution":{"iopub.status.busy":"2022-07-05T03:33:20.280772Z","iopub.execute_input":"2022-07-05T03:33:20.281307Z","iopub.status.idle":"2022-07-05T03:33:20.508516Z","shell.execute_reply.started":"2022-07-05T03:33:20.281276Z","shell.execute_reply":"2022-07-05T03:33:20.507358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Handling Missing values**\n### **Imputation**\n\n**Imputation is technique used for replacing the missing data with some reasonable value(Mean,Median and Mode) to retain most of the data or information of the dataset.**\n\n**Here Im using KNN Imputer, The idea in kNN methods is to identify 'k' samples in the dataset that are similar or close in the space. Then we use these 'k' samples to estimate the value of the missing data points. Each sample's missing values are imputed using the mean value of the 'k'-neighbors found in the dataset. It can give important information to predict the value of our dependent vairable.**","metadata":{"id":"gpji2DIfpCgU"}},{"cell_type":"code","source":"#importing knnimputer from sklearn\nfrom sklearn.impute import KNNImputer\nmy_imputer=KNNImputer(n_neighbors=2)\ndf_train_imputed = pd.DataFrame(my_imputer.fit_transform(df_train_num))\ndf_train_imputed.columns = df_train_num.columns","metadata":{"id":"3cc0b628","execution":{"iopub.status.busy":"2022-07-05T03:33:20.510433Z","iopub.execute_input":"2022-07-05T03:33:20.510730Z","iopub.status.idle":"2022-07-05T03:33:20.561361Z","shell.execute_reply.started":"2022-07-05T03:33:20.510703Z","shell.execute_reply":"2022-07-05T03:33:20.559789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check the distribution of each imputed feature before and after imputation\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (28, 15)})\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(3, 2)\n# Plot the results\nfor feature, fig_pos in zip([\"LotFrontage\", \"GarageYrBlt\", \"MasVnrArea\"], [0, 1, 2]):\n # before imputation\n p = sns.histplot(ax=axes[fig_pos, 0], x=df_train_num[feature],\n kde=True, bins=30, color=\"deepskyblue\", edgecolor=\"black\")\n p.set_xlabel(f\"Before imputation\", fontsize=14)\n\n # after imputation\n q = sns.histplot(ax=axes[fig_pos, 1], x=df_train_imputed[feature],\n kde=True, bins=30, color=\"darkorange\", edgecolor=\"black\")\n q.set_xlabel(f\"After imputation\", fontsize=14)","metadata":{"id":"96231329","outputId":"1fd28e53-ca6a-4912-cc5c-abacdac64007","execution":{"iopub.status.busy":"2022-07-05T03:33:20.563294Z","iopub.execute_input":"2022-07-05T03:33:20.563781Z","iopub.status.idle":"2022-07-05T03:33:22.106307Z","shell.execute_reply.started":"2022-07-05T03:33:20.563743Z","shell.execute_reply":"2022-07-05T03:33:22.104948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**For \"LotFrontage\" distributions is changed after imputations. and others remains the same so we just drop this feature**\n","metadata":{"id":"098056b9"}},{"cell_type":"code","source":"# Drop 'LotFrontage'\ndf_train_imputed.drop([\"LotFrontage\"], axis=1, inplace=True)\ndf_train_imputed.head()","metadata":{"id":"400178b6","outputId":"792cf877-d52b-4961-a07b-2c7291073265","execution":{"iopub.status.busy":"2022-07-05T03:33:22.107863Z","iopub.execute_input":"2022-07-05T03:33:22.108161Z","iopub.status.idle":"2022-07-05T03:33:22.128714Z","shell.execute_reply.started":"2022-07-05T03:33:22.108133Z","shell.execute_reply":"2022-07-05T03:33:22.127959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Test set**","metadata":{"id":"9dcee2e8"}},{"cell_type":"markdown","source":"**The columns that have been deleted in the train set must also be deleted in the test set so that the two data sets remain identical for the modeling and prediction.**","metadata":{"id":"257a60a6"}},{"cell_type":"code","source":"# Drop the same features from test set as for the train set\ndf_test_num.drop([\"LotFrontage\"], axis=1, inplace=True)","metadata":{"id":"ab7f01d7","execution":{"iopub.status.busy":"2022-07-05T03:33:22.129891Z","iopub.execute_input":"2022-07-05T03:33:22.131019Z","iopub.status.idle":"2022-07-05T03:33:22.140541Z","shell.execute_reply.started":"2022-07-05T03:33:22.130978Z","shell.execute_reply":"2022-07-05T03:33:22.139405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the NaN of the test set by ploting percent of missing values per column\ncolumn_with_nan = df_test_num.columns[df_test_num.isnull().any()]\ncolumn_name = []\npercent_nan = []\n\nfor i in column_with_nan:\n column_name.append(i)\n percent_nan.append(\n round(df_test_num[i].isnull().sum()*100/len(df_test_num), 2))\n\ntab = pd.DataFrame(column_name, columns=[\"Column\"])\ntab[\"Percent_NaN\"] = percent_nan\ntab.sort_values(by=[\"Percent_NaN\"], ascending=False, inplace=True)\n\n\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (10, 7)})\nsns.set_style(\"whitegrid\")\n\n# Plot results\np = sns.barplot(x=\"Percent_NaN\", y=\"Column\", data=tab,\n edgecolor=\"black\", color=\"lightskyblue\")\n\np.set_title(\"Percent of NaN per column of the test set\\n\", fontsize=20)\np.set_xlabel(\"\\nPercent of NaN (%)\", fontsize=20)\np.set_ylabel(\"Columns\\n\", fontsize=20)","metadata":{"id":"934c8e62","outputId":"f9fe1b76-307b-40ec-c348-1f4983351611","execution":{"iopub.status.busy":"2022-07-05T03:33:22.142517Z","iopub.execute_input":"2022-07-05T03:33:22.143227Z","iopub.status.idle":"2022-07-05T03:33:22.377984Z","shell.execute_reply.started":"2022-07-05T03:33:22.143168Z","shell.execute_reply":"2022-07-05T03:33:22.376994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Handling Missing values**\n### **Imputation**\n\n**Imputation is technique used for replacing the missing data with some reasonable value(Mean,Median and Mode) to retain most of the data or information of the dataset.**\n\n**Here Im using KNN Imputer, The idea in kNN methods is to identify 'k' samples in the dataset that are similar or close in the space. Then we use these 'k' samples to estimate the value of the missing data points. Each sample's missing values are imputed using the mean value of the 'k'-neighbors found in the dataset. It can give important information to predict the value of our dependent vairable.**","metadata":{"id":"MxIVJRIrjL5Q"}},{"cell_type":"code","source":"#importing knnimputer from sklearn\nfrom sklearn.impute import KNNImputer\nmy_imputer=KNNImputer(n_neighbors=2)\ndf_test_imputed = pd.DataFrame(my_imputer.fit_transform(df_test_num))\ndf_test_imputed.columns = df_test_num.columns","metadata":{"id":"b9b79772","execution":{"iopub.status.busy":"2022-07-05T03:33:22.379589Z","iopub.execute_input":"2022-07-05T03:33:22.379935Z","iopub.status.idle":"2022-07-05T03:33:22.409146Z","shell.execute_reply.started":"2022-07-05T03:33:22.379904Z","shell.execute_reply":"2022-07-05T03:33:22.408085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check the distribution of each imputed feature before and after imputation\n\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (20, 18)})\nsns.set_style(\"whitegrid\")\nfig, axes = plt.subplots(6, 2)\n\n# Plot the results\nfor feature, fig_pos in zip(tab[\"Column\"].tolist(), range(0, 6)):\n\n \"\"\"Features distribution before and after imputation\"\"\"\n\n # before imputation\n p = sns.histplot(ax=axes[fig_pos, 0], x=df_test_num[feature],\n kde=True, bins=30, color=\"deepskyblue\", edgecolor=\"black\")\n p.set_ylabel(f\"Before imputation\", fontsize=14)\n\n # after imputation\n q = sns.histplot(ax=axes[fig_pos, 1], x=df_test_imputed[feature],\n kde=True, bins=30, color=\"darkorange\", edgecolor=\"black\",)\n q.set_ylabel(f\"After imputation\", fontsize=14)","metadata":{"id":"b5ced04c","outputId":"1af2ba60-ef64-48e0-bf74-5e6c3b28f1ce","execution":{"iopub.status.busy":"2022-07-05T03:33:22.410730Z","iopub.execute_input":"2022-07-05T03:33:22.411543Z","iopub.status.idle":"2022-07-05T03:33:25.252436Z","shell.execute_reply.started":"2022-07-05T03:33:22.411501Z","shell.execute_reply":"2022-07-05T03:33:25.251458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The percentage of NaN in each of these fetaures did not exceed 1.5%. Thus, by imputing these missing data, few errors were introduced and the distributions are similar before and after imputation.**","metadata":{"id":"0c430d19"}},{"cell_type":"markdown","source":"## **I.3. Categorical features**","metadata":{"id":"4fdf7589"}},{"cell_type":"markdown","source":"### **I.3.1. Explore and clean Categorical features**","metadata":{"id":"91353294"}},{"cell_type":"code","source":"# Categorical to Quantitative relationship\ncategorical_features = [i for i in df_train.columns if df_train.dtypes[i] == \"object\"]\ncategorical_features.append(\"SalePrice\")\n\n# Train set\ndf_train_categ = df_train[categorical_features]\n\n# Test set (-1 because test set don't have 'Sale Price')\ndf_test_categ = df_test[categorical_features[:-1]]","metadata":{"id":"f9686a85","execution":{"iopub.status.busy":"2022-07-05T03:33:25.254083Z","iopub.execute_input":"2022-07-05T03:33:25.254375Z","iopub.status.idle":"2022-07-05T03:33:25.270346Z","shell.execute_reply.started":"2022-07-05T03:33:25.254344Z","shell.execute_reply":"2022-07-05T03:33:25.269159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Countplot for each of the categorical features in the train set\nfig, axes = plt.subplots(\n round(len(df_train_categ.columns) / 3), 3, figsize=(25, 50))\n\nfor i, ax in enumerate(fig.axes):\n # plot barplot of each feature\n if i < len(df_train_categ.columns) - 1:\n ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=45)\n sns.countplot(\n x=df_train_categ.columns[i], alpha=0.7, data=df_train_categ, ax=ax)\n\nfig.tight_layout()","metadata":{"id":"f192818f","outputId":"56578a66-558d-4ea0-c752-375b02cca7d8","execution":{"iopub.status.busy":"2022-07-05T03:33:25.272482Z","iopub.execute_input":"2022-07-05T03:33:25.272781Z","iopub.status.idle":"2022-07-05T03:33:33.041390Z","shell.execute_reply.started":"2022-07-05T03:33:25.272752Z","shell.execute_reply":"2022-07-05T03:33:33.040104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**By looking above plots we can notice that for some categorical feature the observation are concentrated in a single level of the category. These features are less informative for our model, so it would be better to remove them.**","metadata":{"id":"b0a4bed5"}},{"cell_type":"code","source":"# Drop some categorical 'non-informative' features from train set\ncolumns_to_drop = [\n \"Street\",\n \"Alley\",\n \"LandContour\",\n \"Utilities\",\n \"LandSlope\",\n \"Condition2\",\n \"RoofMatl\",\n \"CentralAir\",\n \"GarageQual\",\n \"GarageCond\",\n \"SaleType\",\n \"PavedDrive\",\n \"LandContour\",\n \"ExterCond\",\n \"GarageCond\",\n \"Heating\",\n \"MiscFeature\",\n \"BsmtFinType2\",\n \"Functional\"\n]\n\n# Train set\ndf_train_categ.drop(columns_to_drop, axis=1, inplace=True)\n\n# Test set\ndf_test_categ.drop(columns_to_drop, axis=1, inplace=True)","metadata":{"id":"6d0cdf67","execution":{"iopub.status.busy":"2022-07-05T03:33:33.042678Z","iopub.execute_input":"2022-07-05T03:33:33.043026Z","iopub.status.idle":"2022-07-05T03:33:33.056485Z","shell.execute_reply.started":"2022-07-05T03:33:33.042995Z","shell.execute_reply":"2022-07-05T03:33:33.055587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With the boxplot we can see the variation of the target 'SalePrice' in each of the categorical features\nfig, axes = plt.subplots(\n round(len(df_train_categ.columns)/3), 3, figsize=(15, 30))\n\nfor i, ax in enumerate(fig.axes):\n # plot the variation of SalePrice in each feature\n if i < len(df_train_categ.columns) - 1:\n ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=45)\n sns.boxplot(\n x=df_train_categ.columns[i], y=\"SalePrice\", data=df_train_categ, ax=ax)\n\nfig.tight_layout()","metadata":{"id":"9dcf8d37","outputId":"8d2fb970-c3da-404b-f7d3-b19e8a875f6a","execution":{"iopub.status.busy":"2022-07-05T03:33:33.057561Z","iopub.execute_input":"2022-07-05T03:33:33.058432Z","iopub.status.idle":"2022-07-05T03:33:40.659061Z","shell.execute_reply.started":"2022-07-05T03:33:33.058358Z","shell.execute_reply":"2022-07-05T03:33:40.657997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Some of these features seem to be codependent such as 'Exterior1st' & 'Exterior2nd', 'BsmtQual' & 'BsmtCond', 'MasVnrType' & 'ExterQual' etc.**\n**So let's plot the contingency table and perform the Chi square test in order to identify these codependency.**","metadata":{"id":"3b4482f0"}},{"cell_type":"code","source":"# Plot contingency table\n\nsns.set(rc={\"figure.figsize\": (10, 7)})\n\nX = [\"Exterior1st\", \"ExterQual\", \"BsmtQual\", \"BsmtQual\", \"BsmtQual\"]\nY = [\"Exterior2nd\", \"MasVnrType\", \"BsmtCond\", \"BsmtExposure\"]\n\nfor i, j in zip(X, Y):\n\n # Contingency table\n cont = df_train_categ[[i, j]].pivot_table(\n index=i, columns=j, aggfunc=len, margins=True, margins_name=\"Total\")\n tx = cont.loc[:, [\"Total\"]]\n ty = cont.loc[[\"Total\"], :]\n n = len(df_train_categ)\n indep = tx.dot(ty) / n\n c = cont.fillna(0) # Replace NaN with 0 in the contingency table\n measure = (c - indep) ** 2 / indep\n xi_n = measure.sum().sum()\n table = measure / xi_n\n \n # Performing Chi-sq test\n CrosstabResult = pd.crosstab(\n index=df_train_categ[i], columns=df_train_categ[j])\n ChiSqResult = chi2_contingency(CrosstabResult)\n # P-Value is the Probability of H0 being True\n print(\n f\"P-Value of the ChiSq Test bewteen {i} and {j} is: {ChiSqResult[1]}\\n\")","metadata":{"id":"246905f6","outputId":"48f87ac9-8ca2-4920-9e6b-c0f1de77320c","execution":{"iopub.status.busy":"2022-07-05T03:33:40.660511Z","iopub.execute_input":"2022-07-05T03:33:40.660772Z","iopub.status.idle":"2022-07-05T03:33:40.784053Z","shell.execute_reply.started":"2022-07-05T03:33:40.660748Z","shell.execute_reply":"2022-07-05T03:33:40.783009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**p-value is significant for all tests so there is some co-dependence between these variables. For this I will drop 'Exterior2nd', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1' and 'MasVnrType'**","metadata":{"id":"b38c94a4"}},{"cell_type":"code","source":"# Let's drop the one of each co-dependent variables\n# Train set\ndf_train_categ.drop(Y, axis=1, inplace=True)\n\n# Test set\ndf_test_categ.drop(Y, axis=1, inplace=True)","metadata":{"id":"ff65580e","execution":{"iopub.status.busy":"2022-07-05T03:33:40.785681Z","iopub.execute_input":"2022-07-05T03:33:40.786136Z","iopub.status.idle":"2022-07-05T03:33:40.794212Z","shell.execute_reply.started":"2022-07-05T03:33:40.786106Z","shell.execute_reply":"2022-07-05T03:33:40.793281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###**I.3.2. Missing data in Categorical features**","metadata":{"id":"410db210"}},{"cell_type":"markdown","source":"### **Train set**","metadata":{"id":"dd752ab9"}},{"cell_type":"code","source":"# Check the NaN of the test set by ploting percent of missing values per column\ncolumn_with_nan = df_train_categ.columns[df_train_categ.isnull().any()]\ncolumn_name = []\npercent_nan = []\n\nfor i in column_with_nan:\n column_name.append(i)\n percent_nan.append(\n round(df_train_categ[i].isnull().sum() * 100 / len(df_train_categ), 2))\n\ntab = pd.DataFrame(column_name, columns=[\"Column\"])\ntab[\"Percent_NaN\"] = percent_nan\ntab.sort_values(by=[\"Percent_NaN\"], ascending=False, inplace=True)\n\n\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (10, 7)})\nsns.set_style(\"whitegrid\")\n\n# Plot results\np = sns.barplot(x=\"Percent_NaN\", y=\"Column\", data=tab,\n edgecolor=\"black\", color=\"deepskyblue\")\np.set_title(\"Percent of NaN per column of the test set\\n\", fontsize=20)\np.set_xlabel(\"\\nPercent of NaN (%)\", fontsize=20)\np.set_ylabel(\"Column Name\\n\", fontsize=20)","metadata":{"id":"3498f096","outputId":"eaa9b3ee-5e30-41bb-dd81-96e0bed18aa6","execution":{"iopub.status.busy":"2022-07-05T03:33:40.804521Z","iopub.execute_input":"2022-07-05T03:33:40.806590Z","iopub.status.idle":"2022-07-05T03:33:41.044848Z","shell.execute_reply.started":"2022-07-05T03:33:40.806554Z","shell.execute_reply":"2022-07-05T03:33:41.043721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Drop the features where the percentage of NaN is higher than 5% to avoid introducing any error. Than impute the NaN of 'BsmtQual' and 'Electrical' by the corresponding modal class**","metadata":{"id":"f4960e0d"}},{"cell_type":"code","source":"# Drop the features where the percentage of NaN is higher than 5%\ndf_train_categ.drop([\"PoolQC\", \"Fence\", \"FireplaceQu\",\n \"GarageType\", \"GarageFinish\"], axis=1, inplace=True,)","metadata":{"id":"026b71bb","execution":{"iopub.status.busy":"2022-07-05T03:33:41.046943Z","iopub.execute_input":"2022-07-05T03:33:41.047275Z","iopub.status.idle":"2022-07-05T03:33:41.055777Z","shell.execute_reply.started":"2022-07-05T03:33:41.047245Z","shell.execute_reply":"2022-07-05T03:33:41.054311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill the NaN of each feature by the corresponding modal class\ncateg_fill_null = {\"BsmtQual\": df_train_categ[\"BsmtQual\"].mode().iloc[0],\n \"BsmtFinType1\": df_train_categ[\"BsmtFinType1\"].mode().iloc[0],\n \"Electrical\": df_train_categ[\"Electrical\"].mode().iloc[0]}\n\ndf_train_categ = df_train_categ.fillna(value=categ_fill_null)","metadata":{"id":"264cb013","execution":{"iopub.status.busy":"2022-07-05T03:33:41.057648Z","iopub.execute_input":"2022-07-05T03:33:41.057983Z","iopub.status.idle":"2022-07-05T03:33:41.070318Z","shell.execute_reply.started":"2022-07-05T03:33:41.057952Z","shell.execute_reply":"2022-07-05T03:33:41.069257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Test set**","metadata":{"id":"0b9ecd36"}},{"cell_type":"markdown","source":"**The columns that have been deleted in the train set must also be deleted in the test set so that the two data sets remain identical for the modeling and prediction.**","metadata":{"id":"68f237ea"}},{"cell_type":"code","source":"# Drop the same features from test set as for the train set\ndf_test_categ.drop([\"PoolQC\", \"Fence\", \"FireplaceQu\",\n \"GarageType\", \"GarageFinish\"], axis=1, inplace=True,)","metadata":{"id":"d56d6e73","execution":{"iopub.status.busy":"2022-07-05T03:33:41.071520Z","iopub.execute_input":"2022-07-05T03:33:41.071849Z","iopub.status.idle":"2022-07-05T03:33:41.078901Z","shell.execute_reply.started":"2022-07-05T03:33:41.071816Z","shell.execute_reply":"2022-07-05T03:33:41.078104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the NaN of the test set by ploting percent of missing values per column\ncolumn_with_nan = df_test_categ.columns[df_test_categ.isnull().any()]\ncolumn_name = []\npercent_nan = []\n\nfor i in column_with_nan:\n column_name.append(i)\n percent_nan.append(\n round(df_test_categ[i].isnull().sum() * 100 / len(df_test_categ), 2))\n\ntab = pd.DataFrame(column_name, columns=[\"Column\"])\ntab[\"Percent_NaN\"] = percent_nan\ntab.sort_values(by=[\"Percent_NaN\"], ascending=False, inplace=True)\n\n\n# Define figure parameters\nsns.set(rc={\"figure.figsize\": (10, 7)})\nsns.set_style(\"whitegrid\")\n\n# Plot results\np = sns.barplot(x=\"Percent_NaN\", y=\"Column\", data=tab,\n edgecolor=\"black\", color=\"deepskyblue\")\np.set_title(\"Percent of NaN per column of the test set\\n\", fontsize=20)\np.set_xlabel(\"\\nPercent of NaN (%)\", fontsize=20)\np.set_ylabel(\"Column Name\\n\", fontsize=20)","metadata":{"id":"6727aeda","outputId":"c738a2e6-ea13-4cd3-c4d0-10b141bfa1b1","execution":{"iopub.status.busy":"2022-07-05T03:33:41.080137Z","iopub.execute_input":"2022-07-05T03:33:41.080634Z","iopub.status.idle":"2022-07-05T03:33:41.300898Z","shell.execute_reply.started":"2022-07-05T03:33:41.080605Z","shell.execute_reply":"2022-07-05T03:33:41.300163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill the NaN of each feature by the corresponding modal class\ncateg_fill_null = {\"BsmtQual\": df_test_categ[\"BsmtQual\"].mode().iloc[0],\n \"BsmtFinType1\": df_test_categ[\"BsmtFinType1\"].mode().iloc[0],\n \"MSZoning\": df_test_categ[\"MSZoning\"].mode().iloc[0],\n \"Exterior1st\": df_test_categ[\"Exterior1st\"].mode().iloc[0],\n \"KitchenQual\": df_test_categ[\"KitchenQual\"].mode().iloc[0]}\n\ndf_test_categ = df_test_categ.fillna(value=categ_fill_null)","metadata":{"id":"60f43e09","execution":{"iopub.status.busy":"2022-07-05T03:33:41.302164Z","iopub.execute_input":"2022-07-05T03:33:41.302647Z","iopub.status.idle":"2022-07-05T03:33:41.314754Z","shell.execute_reply.started":"2022-07-05T03:33:41.302619Z","shell.execute_reply":"2022-07-05T03:33:41.313580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **I.3.3. Encoding Categorical features (using get_dummies)**","metadata":{"id":"1c9114fb"}},{"cell_type":"code","source":"# Train set\nfor i in df_train_categ.columns.tolist()[:-1]:\n df_dummies = pd.get_dummies(df_train_categ[i], prefix=i)\n\n # merge both tables\n df_train_categ = df_train_categ.join(df_dummies)\n\n# Select the binary features only\ndf_train_binary = df_train_categ.iloc[:, 18:]\ndf_train_binary.head()","metadata":{"id":"596325db","outputId":"51257584-e0ae-427d-8aba-0b4de6d909e3","execution":{"iopub.status.busy":"2022-07-05T03:33:41.316051Z","iopub.execute_input":"2022-07-05T03:33:41.316394Z","iopub.status.idle":"2022-07-05T03:33:41.364190Z","shell.execute_reply.started":"2022-07-05T03:33:41.316328Z","shell.execute_reply":"2022-07-05T03:33:41.363097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test set\nfor i in df_test_categ.columns.tolist():\n df_dummies = pd.get_dummies(df_test_categ[i], prefix=i)\n\n # merge both tables\n df_test_categ = df_test_categ.join(df_dummies)\n\n# Select the binary features only\ndf_test_binary = df_test_categ.iloc[:, 17:]\ndf_test_binary.head()","metadata":{"id":"163cc557","outputId":"e8e9e1c9-bbe2-435a-8bd6-ac745c76ef2c","execution":{"iopub.status.busy":"2022-07-05T03:33:41.366210Z","iopub.execute_input":"2022-07-05T03:33:41.366505Z","iopub.status.idle":"2022-07-05T03:33:41.412827Z","shell.execute_reply.started":"2022-07-05T03:33:41.366476Z","shell.execute_reply":"2022-07-05T03:33:41.411663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We can notice that in df_test_binary there is 118 columns while in df_train_binary there is 122. Let's see which columns are missing from df_test_binary**","metadata":{"id":"f1ece3d0"}},{"cell_type":"code","source":"# Let's check if the column headings are the same in both data set, df_train and df_test\ndif_1 = [x for x in df_train_binary.columns if x not in df_test_binary.columns]\nprint(\n f\"Features present in df_train_categ and absent in df_test_categ: {dif_1}\\n\")\n\ndif_2 = [x for x in df_test_binary.columns if x not in df_train_binary.columns]\nprint(\n f\"Features present in df_test_categ set and absent in df_train_categ: {dif_2}\")","metadata":{"id":"9612117b","outputId":"63d1a556-0c68-4285-c181-cc1cacdccfc0","execution":{"iopub.status.busy":"2022-07-05T03:33:41.414053Z","iopub.execute_input":"2022-07-05T03:33:41.414429Z","iopub.status.idle":"2022-07-05T03:33:41.422171Z","shell.execute_reply.started":"2022-07-05T03:33:41.414395Z","shell.execute_reply":"2022-07-05T03:33:41.421355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Four of the binary features are absent from the test set. Thus, these features will be dropped from the train in order to have the same columns in both data set**","metadata":{"id":"e1c20b42"}},{"cell_type":"code","source":"# Let's drop these columns from df_train_binary\ndf_train_binary.drop(dif_1, axis=1, inplace=True)\n\n# Check again if the column headings are the same in both data set\ndif_1 = [x for x in df_train_binary.columns if x not in df_test_binary.columns]\nprint(\n f\"Features present in df_train_categ and absent in df_test_categ: {dif_1}\\n\")\n\ndif_2 = [x for x in df_test_binary.columns if x not in df_train_binary.columns]\nprint(\n f\"Features present in df_test_categ set and absent in df_train_categ: {dif_2}\")","metadata":{"id":"8abb38c7","outputId":"1020b5a5-cc47-4d3c-f88f-1beb9aaa8363","execution":{"iopub.status.busy":"2022-07-05T03:33:41.423450Z","iopub.execute_input":"2022-07-05T03:33:41.423833Z","iopub.status.idle":"2022-07-05T03:33:41.439616Z","shell.execute_reply.started":"2022-07-05T03:33:41.423801Z","shell.execute_reply":"2022-07-05T03:33:41.438763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Both data set have the same featues now**","metadata":{"id":"3b5ea7de"}},{"cell_type":"markdown","source":"##**I.4. Merge numerical and binary features into one data set**","metadata":{"id":"763e8138"}},{"cell_type":"code","source":"# Add binary features to numreical features\n# Train set\ndf_train_new = df_train_imputed.join(df_train_binary)\nprint(f\"Train set: {df_train_new.shape}\")\n\n# Test set\ndf_test_new = df_test_imputed.join(df_test_binary)\nprint(f\"Test set: {df_test_new.shape}\")","metadata":{"id":"563c79e2","outputId":"ffe5b536-f72f-4b4f-c618-83abda8cb6e6","execution":{"iopub.status.busy":"2022-07-05T03:33:41.441598Z","iopub.execute_input":"2022-07-05T03:33:41.441922Z","iopub.status.idle":"2022-07-05T03:33:41.451919Z","shell.execute_reply.started":"2022-07-05T03:33:41.441893Z","shell.execute_reply":"2022-07-05T03:33:41.451039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **I.5. Drop outliers from the train set**","metadata":{"id":"edd492b2"}},{"cell_type":"markdown","source":"**Previoulsy in the part 'I.2. Numerical Features' of this notebook I noticed some houses with large surface (\"GrLivArea\", \"TotalBsmtSF\" and \"GarageArea\") and with a very low Price. It is better for our models to drop theese outliers.**\n \n**I also noticed for both features \"WoodDeckSF\" and \"OpenPorchSF\" a high number of 0 values with a correspondingly high price variation. These outliers should be deleted.**\n**However, since the number of these outliers is very important, the best thing to do is to drop these columns.**","metadata":{"id":"ce6ab37b"}},{"cell_type":"code","source":"sns.set(style=\"darkgrid\") # setting grid to plots\nfig,((ax1,ax2))=plt.subplots(1,2,figsize=(15,4)) # Plotting size of figure\nfig.tight_layout() # Plotting layout of each subplot\nsns.regplot(df_train_new['WoodDeckSF'], df_train_new['SalePrice'],ax=ax1)\nsns.regplot(df_train_new['OpenPorchSF'],df_train_new['SalePrice'] ,ax=ax2)\nplt.title(\"Distribution Before handling missing values\") \nplt.show()","metadata":{"id":"ylxC4XCNbOkV","outputId":"da44aced-7439-454e-d840-22b1ed1879be","execution":{"iopub.status.busy":"2022-07-05T03:33:41.453522Z","iopub.execute_input":"2022-07-05T03:33:41.454152Z","iopub.status.idle":"2022-07-05T03:33:42.387058Z","shell.execute_reply.started":"2022-07-05T03:33:41.454111Z","shell.execute_reply":"2022-07-05T03:33:42.385984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop \"WoodDeckSF\" and \"OpenPorchSF\"\ndf_train_new.drop([\"WoodDeckSF\", \"OpenPorchSF\"], axis=1, inplace=True)\ndf_test_new.drop([\"WoodDeckSF\", \"OpenPorchSF\"], axis=1, inplace=True)","metadata":{"id":"f8640718","execution":{"iopub.status.busy":"2022-07-05T03:33:42.388347Z","iopub.execute_input":"2022-07-05T03:33:42.388640Z","iopub.status.idle":"2022-07-05T03:33:42.399135Z","shell.execute_reply.started":"2022-07-05T03:33:42.388611Z","shell.execute_reply":"2022-07-05T03:33:42.397414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's handle the outliers in \"GrLivArea\", \"TotalBsmtSF\" and \"GarageArea\"\n# Outliers in \"GrLivArea\"\noutliers1 = df_train_new[(df_train_new[\"GrLivArea\"] > 4000) & (\n df_train_new[\"SalePrice\"] <= 200000)].index.tolist()\n\n# Outliers in \"TotalBsmtSF\"\noutliers2 = df_train_new[(df_train_new[\"TotalBsmtSF\"] > 3000) & (\n df_train_new[\"SalePrice\"] <= 400000)].index.tolist()\n\n# Outliers in \"GarageArea\"\noutliers3 = df_train_new[(df_train_new[\"GarageArea\"] > 1200) & (\n df_train_new[\"SalePrice\"] <= 300000)].index.tolist()\n\n# List of all the outliers\noutliers = outliers1 + outliers2 + outliers3\noutliers = list(set(outliers))\nprint(outliers)\n\n# Drop these outlier\ndf_train_new = df_train_new.drop(df_train_new.index[outliers])\n\n# Reset index\ndf_train_new = df_train_new.reset_index().drop(\"index\", axis=1)","metadata":{"id":"WNIyPDj7IqX2","outputId":"9fafe8c9-adea-41fb-fb59-2ea0d91031f5","execution":{"iopub.status.busy":"2022-07-05T03:33:42.400924Z","iopub.execute_input":"2022-07-05T03:33:42.401253Z","iopub.status.idle":"2022-07-05T03:33:42.416652Z","shell.execute_reply.started":"2022-07-05T03:33:42.401218Z","shell.execute_reply":"2022-07-05T03:33:42.415698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#**II. Feature engineering**","metadata":{"id":"2dd74013"}},{"cell_type":"markdown","source":"**While looking closley at the remaining features, we can notice that several of them designate a given surface of the property. Thus, I will try to combine some of these surfaces into indicators without losing the information they provide.**\n \n**In addition, I will turn years into age, e.g. year of construction will be transformed into age of the house since the construction.**","metadata":{"id":"839ca38c"}},{"cell_type":"code","source":"# Define a function to calculate the occupancy rate of the first floor of the total living area\n\ndef floor_occupation(x):\n \"\"\"First floor occupation of the total live area\n\n floor_occupation equation has the following form:\n (1st Floor Area * 100) / (Ground Live Area)\n Args:\n x -- the corresponding feature\n\n Returns:\n 0 -- if Ground Live Area = 0\n equation -- if Ground Live Area > 0\n \"\"\"\n if x[\"GrLivArea\"] == 0:\n return 0\n else:\n return x[\"1stFlrSF\"] * 100 / x[\"GrLivArea\"]\n\n\n# Apply the function on train and test set\ndf_train_new[\"1stFlrPercent\"] = df_train_new.apply(\n lambda x: floor_occupation(x), axis=1)\n\ndf_test_new[\"1stFlrPercent\"] = df_test_new.apply(\n lambda x: floor_occupation(x), axis=1)\n\n# Drop \"1stFlrSF\" and \"2ndFlrSF\"\ndf_train_new.drop([\"1stFlrSF\", \"2ndFlrSF\"], axis=1, inplace=True)\ndf_test_new.drop([\"1stFlrSF\", \"2ndFlrSF\"], axis=1, inplace=True)","metadata":{"id":"9df678b0","execution":{"iopub.status.busy":"2022-07-05T03:33:42.417905Z","iopub.execute_input":"2022-07-05T03:33:42.418460Z","iopub.status.idle":"2022-07-05T03:33:42.517809Z","shell.execute_reply.started":"2022-07-05T03:33:42.418430Z","shell.execute_reply":"2022-07-05T03:33:42.516563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a function to calculate the occupancy rate of the finished basement area\n\n\ndef bsmt_finish(x):\n \"\"\"Propotion of finished area in basement \n\n bsmt_finish equation has the following form:\n (Finished Basement Area * 100) / (Total Basement Area)\n\n Args:\n x -- the corresponding feature\n\n Returns:\n 0 -- if Total Basement Area = 0\n equation -- if Total Basement Area > 0\n \"\"\"\n if x[\"TotalBsmtSF\"] == 0:\n return 0\n else:\n return x[\"BsmtFinSF1\"] * 100 / x[\"TotalBsmtSF\"]\n\n\n# Apply the function on train and test set\ndf_train_new[\"BsmtFinPercent\"] = df_train_new.apply(\n lambda x: bsmt_finish(x), axis=1)\n\ndf_test_new[\"BsmtFinPercent\"] = df_test_new.apply(\n lambda x: bsmt_finish(x), axis=1)\n\n# Drop \"BsmtFinSF1\"\ndf_train_new.drop([\"BsmtFinSF1\"], axis=1, inplace=True)\ndf_test_new.drop([\"BsmtFinSF1\"], axis=1, inplace=True)","metadata":{"id":"863217cb","execution":{"iopub.status.busy":"2022-07-05T03:33:42.519131Z","iopub.execute_input":"2022-07-05T03:33:42.519542Z","iopub.status.idle":"2022-07-05T03:33:42.620217Z","shell.execute_reply.started":"2022-07-05T03:33:42.519494Z","shell.execute_reply":"2022-07-05T03:33:42.619158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert Year of construction to Age of the house since the construction\ndf_train_new[\"AgeSinceConst\"] = (\n df_train_new[\"YearBuilt\"].max() - df_train_new[\"YearBuilt\"])\n\ndf_test_new[\"AgeSinceConst\"] = df_test_new[\"YearBuilt\"].max() - \\\n df_test_new[\"YearBuilt\"]\n\n# Drop \"YearBuilt\"\ndf_train_new.drop([\"YearBuilt\"], axis=1, inplace=True)\ndf_test_new.drop([\"YearBuilt\"], axis=1, inplace=True)","metadata":{"id":"1d4e0fab","execution":{"iopub.status.busy":"2022-07-05T03:33:42.621596Z","iopub.execute_input":"2022-07-05T03:33:42.621903Z","iopub.status.idle":"2022-07-05T03:33:42.634609Z","shell.execute_reply.started":"2022-07-05T03:33:42.621872Z","shell.execute_reply":"2022-07-05T03:33:42.633581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert Year of remodeling to Age of the house since the remodeling\ndf_train_new[\"AgeSinceRemod\"] = (\n df_train_new[\"YearRemodAdd\"].max() - df_train_new[\"YearRemodAdd\"])\n\ndf_test_new[\"AgeSinceRemod\"] = (\n df_test_new[\"YearRemodAdd\"].max() - df_test_new[\"YearRemodAdd\"])\n\n# Drop \"YearRemodAdd\"\ndf_train_new.drop([\"YearRemodAdd\"], axis=1, inplace=True)\ndf_test_new.drop([\"YearRemodAdd\"], axis=1, inplace=True)","metadata":{"id":"91d8f0b3","execution":{"iopub.status.busy":"2022-07-05T03:33:42.635745Z","iopub.execute_input":"2022-07-05T03:33:42.636644Z","iopub.status.idle":"2022-07-05T03:33:42.647980Z","shell.execute_reply.started":"2022-07-05T03:33:42.636602Z","shell.execute_reply":"2022-07-05T03:33:42.646948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**To avoid redundancy and to mitigate the strong variations of some features according to the SalePrice. I will use a Boxcox transformation for skewed features. Generally in real estate, as the area of the property increases, the price per square feet decreases, hence the use of Boxcox will giving good accuracy with low RMSC more than log transfermation.**","metadata":{"id":"d152cb51"}},{"cell_type":"code","source":"continuous_features = [\"OverallQual\", \"TotalBsmtSF\", \"GrLivArea\",\n \"FullBath\", \"TotRmsAbvGrd\", \"GarageCars\", \"GarageArea\",\n \"MasVnrArea\", \"Fireplaces\", \"1stFlrPercent\",\n \"BsmtFinPercent\", \"AgeSinceConst\", \"AgeSinceRemod\"]\ndf_skew_verify = df_train_new.loc[:, continuous_features]","metadata":{"id":"609d3003","execution":{"iopub.status.busy":"2022-07-05T03:33:42.649261Z","iopub.execute_input":"2022-07-05T03:33:42.649995Z","iopub.status.idle":"2022-07-05T03:33:42.657136Z","shell.execute_reply.started":"2022-07-05T03:33:42.649963Z","shell.execute_reply":"2022-07-05T03:33:42.655718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select features with absolute Skew higher than 0.5\nskew_ft = []\n\nfor i in continuous_features:\n # list of skew for each corresponding feature\n skew_ft.append(abs(df_skew_verify[i].skew()))\n\ndf_skewed = pd.DataFrame({\"Columns\": continuous_features,\n \"Abs_Skew\": skew_ft})\n\nsk_features = df_skewed[df_skewed[\"Abs_Skew\"] > 0.5][\"Columns\"].tolist()\nprint(f\"List of skewed features: {sk_features}\")","metadata":{"id":"4e83d154","outputId":"e5af7a9e-7807-4236-81c2-bf981c200b56","execution":{"iopub.status.busy":"2022-07-05T03:33:42.658812Z","iopub.execute_input":"2022-07-05T03:33:42.659098Z","iopub.status.idle":"2022-07-05T03:33:42.677940Z","shell.execute_reply.started":"2022-07-05T03:33:42.659069Z","shell.execute_reply":"2022-07-05T03:33:42.676561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Log transformation of the skewed features\n#sf_features = [\"TotalBsmtSF\", \"GrLivArea\", \"MasVnrArea\", \"GarageArea\"]\n\nfor i in sk_features:\n # loop over i (features) to calculate Log of surfaces\n # Train set\n df_train_new[i] = np.log((df_train_new[i])+1)\n \n # Test set\n df_test_new[i] = np.log((df_test_new[i])+1)","metadata":{"id":"5d242b48","execution":{"iopub.status.busy":"2022-07-05T03:33:42.679858Z","iopub.execute_input":"2022-07-05T03:33:42.680226Z","iopub.status.idle":"2022-07-05T03:33:42.692826Z","shell.execute_reply.started":"2022-07-05T03:33:42.680164Z","shell.execute_reply":"2022-07-05T03:33:42.691332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **III. Preparing data for modeling**","metadata":{"id":"b8a6e4eb"}},{"cell_type":"markdown","source":"## **III.1. Target variable 'SalePrice'**","metadata":{"id":"280c6386"}},{"cell_type":"markdown","source":"### **Guassian Transformation**\n\n**Some machine learning algorithms like linear and logistic assume that the features are normally distributed -Accuracy -Performance.These are different type of transformations,**\n\n* **logarithmic transformation**\n\n* **reciprocal transformation**\n\n* **square root transformation**\n\n* **exponential transformation** \n\n* **boxcox transformation** (***Here im choossing BoxCox transformation for better accuracy with low RMSC***) \n\n**BoxCOx Transformation: T(Y)=(Y exp(λ)−1)/λ**\n\n**where Y is the response variable and λ is the transformation parameter. λ varies from -5 to 5. In the transformation, all values of λ are considered and the optimal value for a given variable is selected.\nBoxcox transformation of the independent variable ('SalePrice') to have a distribution that approaches the normal distribution**","metadata":{"id":"ffe23d4f"}},{"cell_type":"code","source":"# Log transformation of the target variable \"SalePrice\"\n#applying lock trasformation, see ther distibution\nimport scipy.stats as stats\n\ndf_train_new['SalePriceLog'],parameters=stats.boxcox(df_train_new['SalePrice']) \n\n# Plot the distribution before and after transformation\nfig, axes = plt.subplots(1, 2)\nfig.suptitle(\"Distribution of 'SalePrice' before and after boxcox-transformation\")\n\n# before log transformation\np = sns.histplot(ax=axes[0], x=df_train_new[\"SalePrice\"],\n kde=True, bins=100, color=\"deepskyblue\")\np.set_xlabel(\"SalePrice\", fontsize=16)\np.set_ylabel(\"Effectif\", fontsize=16)\n\n# after log transformation\nq = sns.histplot(ax=axes[1], x=df_train_new[\"SalePriceLog\"],\n kde=True, bins=100, color=\"darkorange\")\nq.set_xlabel(\"SalePriceLog\", fontsize=16)\nq.set_ylabel(\"\", fontsize=16)\n# after log transformation\n","metadata":{"id":"2b157034","outputId":"9e23faa4-34e2-4672-9366-16ab044fa15b","execution":{"iopub.status.busy":"2022-07-05T03:33:42.695357Z","iopub.execute_input":"2022-07-05T03:33:42.696614Z","iopub.status.idle":"2022-07-05T03:33:43.453596Z","shell.execute_reply.started":"2022-07-05T03:33:42.696580Z","shell.execute_reply":"2022-07-05T03:33:43.452522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop the original SalePrice\ndf_train_new.drop([\"SalePrice\"], axis=1, inplace=True)","metadata":{"id":"9f2dba0c","execution":{"iopub.status.busy":"2022-07-05T03:33:43.454841Z","iopub.execute_input":"2022-07-05T03:33:43.455122Z","iopub.status.idle":"2022-07-05T03:33:43.461914Z","shell.execute_reply.started":"2022-07-05T03:33:43.455094Z","shell.execute_reply":"2022-07-05T03:33:43.461120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **III.2. Train Test Split Data and Standardization**","metadata":{"id":"c4664537"}},{"cell_type":"code","source":"# Extract the features (X) and the target (y)\n# Features (X)\nX = df_train_new[[i for i in list(\n df_train_new.columns) if i != \"SalePriceLog\"]]\nprint(X.shape)\n\n# Target (y)\ny = df_train_new.loc[:, \"SalePriceLog\"]\nprint(y.shape)","metadata":{"id":"b8d71440","outputId":"aa49b561-1df8-4f3a-89f9-c3fec5e72a9a","execution":{"iopub.status.busy":"2022-07-05T03:33:43.462829Z","iopub.execute_input":"2022-07-05T03:33:43.463093Z","iopub.status.idle":"2022-07-05T03:33:43.490756Z","shell.execute_reply.started":"2022-07-05T03:33:43.463067Z","shell.execute_reply":"2022-07-05T03:33:43.489787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split into X_train and X_test (by stratifying on y)\n# Stratify on a continuous variable by splitting it in bins\n# Create the bins.\nbins = np.linspace(0, len(y), 150)\ny_binned = np.digitize(y, bins)\n\n# Split the data\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,\n stratify=y_binned, shuffle=True)\nprint(f\"X_train:{X_train.shape}\\ny_train:{y_train.shape}\")\nprint(f\"\\nX_test:{X_test.shape}\\ny_test:{y_test.shape}\")","metadata":{"id":"8b1d9e80","outputId":"b3c5707f-0eac-4e39-bf0a-22e4ed4760f7","execution":{"iopub.status.busy":"2022-07-05T03:33:43.492071Z","iopub.execute_input":"2022-07-05T03:33:43.493712Z","iopub.status.idle":"2022-07-05T03:33:43.506447Z","shell.execute_reply.started":"2022-07-05T03:33:43.493680Z","shell.execute_reply":"2022-07-05T03:33:43.505259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize the data\nstd_scale = preprocessing.StandardScaler().fit(X_train)\nX_train = std_scale.transform(X_train)\nX_test = std_scale.transform(X_test)\n# The same standardization is applied for df_test_new\ndf_test_new = std_scale.transform(df_test_new)\n\n# The output of standardization is a vector. Let's turn it into a table\n# Convert X, y and test data into dataframe\nX_train = pd.DataFrame(X_train, columns=X.columns)\nX_test = pd.DataFrame(X_test, columns=X.columns)\ndf_test_new = pd.DataFrame(df_test_new, columns=X.columns)\n\ny_train = pd.DataFrame(y_train)\ny_train = y_train.reset_index().drop(\"index\", axis=1)\n\ny_test = pd.DataFrame(y_test)\ny_test = y_test.reset_index().drop(\"index\", axis=1)","metadata":{"id":"67ae2c67","execution":{"iopub.status.busy":"2022-07-05T03:33:43.509392Z","iopub.execute_input":"2022-07-05T03:33:43.509640Z","iopub.status.idle":"2022-07-05T03:33:43.535037Z","shell.execute_reply.started":"2022-07-05T03:33:43.509617Z","shell.execute_reply":"2022-07-05T03:33:43.534246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"I-uy9_TlnDah"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **III.3. Backward Stepwise Regression**\n\n\n**Backward Stepwise Regression at each step gradually eliminates variables from the regression model to find a reduced model that best explains the data. Also known as Backward Elimination regression.**\n","metadata":{"id":"nMEcdU1etFdi"}},{"cell_type":"code","source":"Selected_Features = []\n\n\ndef backward_regression(X, y, initial_list=[], threshold_in=0.01, threshold_out=0.05, verbose=True):\n \"\"\"To select feature with Backward Stepwise Regression \n\n Args:\n X -- features values\n y -- target variable\n initial_list -- features header\n threshold_in -- pvalue threshold of features to keep\n threshold_out -- pvalue threshold of features to drop\n verbose -- true to produce lots of logging output\n\n Returns:\n list of selected features for modeling \n \"\"\"\n included = list(X.columns)\n while True:\n changed = False\n model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included]))).fit()\n # use all coefs except intercept\n pvalues = model.pvalues.iloc[1:]\n worst_pval = pvalues.max() # null if pvalues is empty\n if worst_pval > threshold_out:\n changed = True\n worst_feature = pvalues.idxmax()\n included.remove(worst_feature)\n if verbose:\n print(f\"worst_feature : {worst_feature}, {worst_pval} \")\n if not changed:\n break\n Selected_Features.append(included)\n print(f\"\\nSelected Features:\\n{Selected_Features[0]}\")\n\n\n# Application of the backward regression function on our training data\nbackward_regression(X_train, y_train)","metadata":{"id":"Jzc_6g95mzTU","outputId":"5f6ffe74-7d0e-4dff-99b2-20c3788db9fa","execution":{"iopub.status.busy":"2022-07-05T03:33:43.536211Z","iopub.execute_input":"2022-07-05T03:33:43.537209Z","iopub.status.idle":"2022-07-05T03:33:46.162879Z","shell.execute_reply.started":"2022-07-05T03:33:43.537158Z","shell.execute_reply":"2022-07-05T03:33:46.162015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Keep the selected features only\nX_train = X_train.loc[:, Selected_Features[0]]\nX_test = X_test.loc[:, Selected_Features[0]]\ndf_test_new = df_test_new.loc[:, Selected_Features[0]]","metadata":{"id":"UdgqMAY-mzKk","execution":{"iopub.status.busy":"2022-07-05T03:33:46.164399Z","iopub.execute_input":"2022-07-05T03:33:46.164970Z","iopub.status.idle":"2022-07-05T03:33:46.173627Z","shell.execute_reply.started":"2022-07-05T03:33:46.164933Z","shell.execute_reply":"2022-07-05T03:33:46.172645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **III.4. Cook distance**","metadata":{"id":"75927ad8"}},{"cell_type":"markdown","source":"**Using Cook distance we can detects data with large residuals (outliers) that can distort the prediction and the accuracy of a regression.**","metadata":{"id":"409e0f8b"}},{"cell_type":"code","source":"X_constant = sm.add_constant(X_train)\n\nmodel = sm.OLS(y_train, X_constant)\nlr = model.fit()\n\n# Cook distance\nnp.set_printoptions(suppress=True)\n\n# Create an instance of influence\ninfluence = lr.get_influence()\n\n# Get Cook's distance for each observation\ncooks = influence.cooks_distance\n\n# Result as a dataframe\ncook_df = pd.DataFrame({\"Cook_Distance\": cooks[0], \"p_value\": cooks[1]})\ncook_df.head()","metadata":{"id":"b7973376","outputId":"2cd07dd3-7420-40ce-c717-436b4d1b11da","execution":{"iopub.status.busy":"2022-07-05T03:33:46.175600Z","iopub.execute_input":"2022-07-05T03:33:46.177602Z","iopub.status.idle":"2022-07-05T03:33:46.213263Z","shell.execute_reply.started":"2022-07-05T03:33:46.177495Z","shell.execute_reply":"2022-07-05T03:33:46.212384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove the influential observation from X_train and y_train\ninfluent_observation = cook_df[cook_df[\"p_value\"] < 0.05].index.tolist()\nprint(f\"Influential observations dropped: {influent_observation}\")\n\n# Drop these obsrevations\nX_train = X_train.drop(X_train.index[influent_observation])\ny_train = y_train.drop(y_train.index[influent_observation])","metadata":{"id":"db96a6f0","outputId":"3d5a2f40-b624-4266-ee56-5e2933e837ed","execution":{"iopub.status.busy":"2022-07-05T03:33:46.214821Z","iopub.execute_input":"2022-07-05T03:33:46.215411Z","iopub.status.idle":"2022-07-05T03:33:46.226089Z","shell.execute_reply.started":"2022-07-05T03:33:46.215373Z","shell.execute_reply":"2022-07-05T03:33:46.224988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **IV. Modeling**","metadata":{"id":"3d4a9664"}},{"cell_type":"markdown","source":"## **IV.1. Models, metrics selection**","metadata":{"id":"e5e6a156"}},{"cell_type":"markdown","source":"**Here I am going to use RMSE and R² metrics in order to measure the performance of the selected models and their predictions.**\n \n**Then I will test the models that best meet the estimation of house prices. It's clearly a regression so here are the following models I will use:**\n- **Ridge regression**\n- **Lasso regression**\n- **Elastic Net regression**\n- **Support Vector regression (SVR)**\n- **K Nearest Kneighbour**\n- **Decision Tree**\n- **Random Forest regression**\n- **XGBoost**\n* **LigthGBM**\n ","metadata":{"id":"484f49f9"}},{"cell_type":"code","source":"import sklearn\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import ElasticNet\nfrom sklearn.svm import SVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor","metadata":{"id":"4e1edd32","execution":{"iopub.status.busy":"2022-07-05T03:33:46.228329Z","iopub.execute_input":"2022-07-05T03:33:46.229135Z","iopub.status.idle":"2022-07-05T03:33:46.236728Z","shell.execute_reply.started":"2022-07-05T03:33:46.229068Z","shell.execute_reply":"2022-07-05T03:33:46.235688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defining function for each metrics\n\n# R²_score\ndef rsqr_score(test, pred):\n r2_ = r2_score(test, pred)\n return r2_\n\n# RMSE\ndef rmse_score(test, pred):\n rmse_ = np.sqrt(mean_squared_error(test, pred))\n return rmse_\n\n# Print the scores\ndef print_score(test, pred): \n print(f\"- Regressor: {regr.__class__.__name__}\")\n print(f\"R²: {rsqr_score(test, pred)}\")\n print(f\"RMSE: {rmse_score(test, pred)}\\n\")\n # Args:\n # test -- test data\n # pred -- predicted data\n # Returns:\n # print the regressor name\n # print the R squared score\n # print Root Mean Square Error score","metadata":{"id":"50a56a9b","execution":{"iopub.status.busy":"2022-07-05T03:33:46.238491Z","iopub.execute_input":"2022-07-05T03:33:46.239144Z","iopub.status.idle":"2022-07-05T03:33:46.248307Z","shell.execute_reply.started":"2022-07-05T03:33:46.239106Z","shell.execute_reply":"2022-07-05T03:33:46.247240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define regression models\nridge = Ridge()\nlasso = Lasso(alpha=0.001)\nelastic = ElasticNet(alpha=0.001)\nsvr = SVR()\nknn= KNeighborsRegressor()\nrdf = RandomForestRegressor()\ndc = DecisionTreeRegressor()\nxgboost = XGBRegressor()\nlgbm = LGBMRegressor()\n\n\n# Train models on X_train and y_train\nfor regr in [ridge, lasso, elastic, svr, knn, dc, rdf, xgboost, lgbm]:\n # fit the corresponding model\n regr.fit(X_train, y_train)\n y_pred = regr.predict(X_test)\n # Print the defined metrics above for each classifier\n print_score(y_test, y_pred)","metadata":{"id":"0d2693d0","outputId":"aee6dc20-299d-4836-9d6c-3466a677d459","execution":{"iopub.status.busy":"2022-07-05T03:33:46.250256Z","iopub.execute_input":"2022-07-05T03:33:46.250896Z","iopub.status.idle":"2022-07-05T03:33:47.861641Z","shell.execute_reply.started":"2022-07-05T03:33:46.250849Z","shell.execute_reply":"2022-07-05T03:33:47.860789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**According to the result of R² and Root Mean Squared of these 9 models, we can conclude that the relationship between the features and the target variable is clearly linear.**","metadata":{"id":"fcefdedf"}},{"cell_type":"markdown","source":"##**IV.2. Hyperparameters tuning and model optimization**","metadata":{"id":"f6d3bf08"}},{"cell_type":"markdown","source":"###**IV.2.1. Ridge regression**","metadata":{"id":"07557b0b"}},{"cell_type":"markdown","source":"**Ridge will reduce the impact of features that are not important in predicting the target values.**\n \n**[Click here for more information about Ridge regression](https://medium.com/@vijay.swamy1/lasso-versus-ridge-versus-elastic-net-1d57cfc64b58)**\n","metadata":{"id":"b8391c20"}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\n# Define hyperparameters\nalphas = np.logspace(-5, 5, 50).tolist()\n\ntuned_parameters = {\"alpha\": alphas}\n\n# GridSearch\nridge_cv = GridSearchCV(Ridge(), tuned_parameters, cv=10, n_jobs=-1, verbose=1)\n\n# fit the GridSearch on train set\nridge_cv.fit(X_train, y_train)\n\n# print best params and the corresponding R²\nprint(f\"Best hyperparameters: {ridge_cv.best_params_}\")\nprint(f\"Best R² (train): {ridge_cv.best_score_}\")","metadata":{"id":"ccd852d5","outputId":"6f116707-dc36-4450-c46e-ae89259aabb8","execution":{"iopub.status.busy":"2022-07-05T03:33:47.863133Z","iopub.execute_input":"2022-07-05T03:33:47.863722Z","iopub.status.idle":"2022-07-05T03:33:49.726564Z","shell.execute_reply.started":"2022-07-05T03:33:47.863680Z","shell.execute_reply":"2022-07-05T03:33:49.725676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ridge Regressor with the best hyperparameters\nridge_mod = Ridge(alpha=ridge_cv.best_params_[\"alpha\"])\n\n# Fit the model on train set\nridge_mod.fit(X_train, y_train)\n\n# Predict on test set\ny_pred = ridge_mod.predict(X_test)\n\nprint(f\"- {ridge_mod.__class__.__name__}\")\nprint(f\"R²: {rsqr_score(y_test, y_pred)}\")\nprint(f\"RMSE: {rmse_score(y_test, y_pred)}\")","metadata":{"id":"86d5eb33","outputId":"6360c71c-7000-4333-84ba-f3ea06fef20b","execution":{"iopub.status.busy":"2022-07-05T03:33:49.730569Z","iopub.execute_input":"2022-07-05T03:33:49.732691Z","iopub.status.idle":"2022-07-05T03:33:49.753001Z","shell.execute_reply.started":"2022-07-05T03:33:49.732654Z","shell.execute_reply":"2022-07-05T03:33:49.751974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model results into lists\nmodel_list = []\nr2_list = []\nrmse_list = []\n\nmodel_list.append(ridge_mod.__class__.__name__)\nr2_list.append(round(rsqr_score(y_test, y_pred), 4))\nrmse_list.append(round(rmse_score(y_test, y_pred), 4))","metadata":{"id":"3cff4b81","execution":{"iopub.status.busy":"2022-07-05T03:33:49.754542Z","iopub.execute_input":"2022-07-05T03:33:49.755116Z","iopub.status.idle":"2022-07-05T03:33:49.772038Z","shell.execute_reply.started":"2022-07-05T03:33:49.755074Z","shell.execute_reply":"2022-07-05T03:33:49.770788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Actual vs. Predicted house prices\nactual_price = np.exp(y_test[\"SalePriceLog\"])\npredicted_price = np.exp(y_pred)\n\nplt.figure()\nplt.title(\"Actual vs. Predicted house prices\\n (Ridge)\", fontsize=20)\nplt.scatter(actual_price, predicted_price,\n color=\"deepskyblue\", marker=\"o\", facecolors=\"none\")\nplt.plot([0, 800000], [0, 800000], \"darkorange\", lw=2)\nplt.xlim(0, 800000)\nplt.ylim(0, 800000)\nplt.xlabel(\"\\nActual Price\", fontsize=16)\nplt.ylabel(\"Predicted Price\\n\", fontsize=16)\nplt.show()","metadata":{"id":"9fdfb045","outputId":"75bd547f-4041-4cdb-83c4-2bf67b8f5efb","execution":{"iopub.status.busy":"2022-07-05T03:33:49.773685Z","iopub.execute_input":"2022-07-05T03:33:49.774122Z","iopub.status.idle":"2022-07-05T03:33:50.049209Z","shell.execute_reply.started":"2022-07-05T03:33:49.774079Z","shell.execute_reply":"2022-07-05T03:33:50.048043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **IV.2.2. Lasso regression**","metadata":{"id":"481d95f2"}},{"cell_type":"markdown","source":"**Lasso will eliminate many features, and reduce overfitting in the linear model.**\n\n**[Click here for more information about Lasso regression](https://medium.com/@vijay.swamy1/lasso-versus-ridge-versus-elastic-net-1d57cfc64b58)**","metadata":{"id":"f35c817c"}},{"cell_type":"code","source":"# Define hyperparameters\nalphas = np.logspace(-5, 5, 50).tolist()\n\ntuned_parameters = {\"alpha\": alphas}\n\n# GridSearch\nlasso_cv = GridSearchCV(Lasso(), tuned_parameters, cv=10, n_jobs=-1, verbose=1)\n\n# fit the GridSearch on train set\nlasso_cv.fit(X_train, y_train)\n\n# print best params and the corresponding R²\nprint(f\"Best hyperparameters: {lasso_cv.best_params_}\")\nprint(f\"Best R² (train): {lasso_cv.best_score_}\")","metadata":{"id":"99dddb74","outputId":"dd681a88-389c-440e-ea73-c26b0c2ab980","execution":{"iopub.status.busy":"2022-07-05T03:33:50.050753Z","iopub.execute_input":"2022-07-05T03:33:50.051044Z","iopub.status.idle":"2022-07-05T03:33:52.127068Z","shell.execute_reply.started":"2022-07-05T03:33:50.051016Z","shell.execute_reply":"2022-07-05T03:33:52.126237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lasso Regressor with the best hyperparameters\nlasso_mod = Lasso(alpha=lasso_cv.best_params_[\"alpha\"])\n\n# Fit the model on train set\nlasso_mod.fit(X_train, y_train)\n\n# Predict on test set\ny_pred = lasso_mod.predict(X_test)\n\nprint(f\"- {lasso_mod.__class__.__name__}\")\nprint(f\"R²: {rsqr_score(y_test, y_pred)}\")\nprint(f\"RMSE: {rmse_score(y_test, y_pred)}\")","metadata":{"id":"c3dc9314","outputId":"54bdd1f1-ae2e-4eeb-948a-dfb413968f3f","execution":{"iopub.status.busy":"2022-07-05T03:33:52.128634Z","iopub.execute_input":"2022-07-05T03:33:52.129246Z","iopub.status.idle":"2022-07-05T03:33:52.157398Z","shell.execute_reply.started":"2022-07-05T03:33:52.129208Z","shell.execute_reply":"2022-07-05T03:33:52.156532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model results into lists\nmodel_list.append(lasso_mod.__class__.__name__)\nr2_list.append(round(rsqr_score(y_test, y_pred), 4))\nrmse_list.append(round(rmse_score(y_test, y_pred), 4))","metadata":{"id":"31e70794","execution":{"iopub.status.busy":"2022-07-05T03:33:52.158963Z","iopub.execute_input":"2022-07-05T03:33:52.159562Z","iopub.status.idle":"2022-07-05T03:33:52.174611Z","shell.execute_reply.started":"2022-07-05T03:33:52.159525Z","shell.execute_reply":"2022-07-05T03:33:52.173634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Actual vs. Predicted house prices\nactual_price = np.exp(y_test[\"SalePriceLog\"])\npredicted_price = np.exp(y_pred)\n\nplt.figure()\nplt.title(\"Actual vs. Predicted house prices\\n (Lasso)\", fontsize=20)\nplt.scatter(actual_price, predicted_price,\n color=\"deepskyblue\", marker=\"o\", facecolors=\"none\")\nplt.plot([0, 800000], [0, 800000], \"darkorange\", lw=2)\nplt.xlim(0, 800000)\nplt.ylim(0, 800000)\nplt.xlabel(\"\\nActual Price\", fontsize=16)\nplt.ylabel(\"Predicted Price\\n\", fontsize=16)\nplt.show()","metadata":{"id":"c5c6df4f","outputId":"eda9c38e-bd0a-4365-b52b-a8f5fa7a8bec","execution":{"iopub.status.busy":"2022-07-05T03:33:52.176645Z","iopub.execute_input":"2022-07-05T03:33:52.177925Z","iopub.status.idle":"2022-07-05T03:33:52.445670Z","shell.execute_reply.started":"2022-07-05T03:33:52.177890Z","shell.execute_reply":"2022-07-05T03:33:52.444198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **IV.2.3. XGBoost regression**","metadata":{"id":"4023ab72"}},{"cell_type":"markdown","source":"**XGBoost is one of the most popular algorithms that are based on Gradient Boosted Machines. Gradient Boosting refers to a methodology where an ensemble of weak learners is used to improve the model performance in terms of efficiency, accuracy, and interpretability. Gradient Boosting can be applied to a regression by taking the average of the outputs by the weak learners.**\n\n**[Click here for more information about XGBoost](https://neptune.ai/blog/xgboost-vs-lightgbm)**","metadata":{"id":"99d7ed3b"}},{"cell_type":"code","source":"# Define hyperparameters\ntuned_parameters = {\"max_depth\": [3],\n \"colsample_bytree\": [0.3, 0.7],\n \"learning_rate\": [0.01, 0.05, 0.1],\n \"n_estimators\": [100, 500]}\n\n# GridSearch\nxgbr_cv = GridSearchCV(estimator=XGBRegressor(),\n param_grid=tuned_parameters,\n cv=5,\n n_jobs=-1,\n verbose=1)\n\n# fit the GridSearch on train set\nxgbr_cv.fit(X_train, y_train)\n\n# print best params and the corresponding R²\nprint(f\"Best hyperparameters: {xgbr_cv.best_params_}\\n\")\nprint(f\"Best R²: {xgbr_cv.best_score_}\")","metadata":{"id":"ab12752c","outputId":"3cde33a7-f113-493f-a474-011df1b180ca","execution":{"iopub.status.busy":"2022-07-05T03:33:52.447452Z","iopub.execute_input":"2022-07-05T03:33:52.447796Z","iopub.status.idle":"2022-07-05T03:34:10.572840Z","shell.execute_reply.started":"2022-07-05T03:33:52.447766Z","shell.execute_reply":"2022-07-05T03:34:10.572014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**It's always better to set more hyperparameters with more cross validation but it's very time consuming with XGBRegressor in kaggle kerner. Thus, I limited the tuning to 3 hyperparameters with 5 cross validation only.**","metadata":{"id":"f485019a"}},{"cell_type":"code","source":"# XGB Regressor with the best hyperparameters\nxgbr_mod = XGBRegressor(seed=20,\n colsample_bytree=xgbr_cv.best_params_[\"colsample_bytree\"],\n learning_rate=xgbr_cv.best_params_[\"learning_rate\"],\n max_depth=xgbr_cv.best_params_[\"max_depth\"],\n n_estimators=xgbr_cv.best_params_[\"n_estimators\"])\n\n# Fit the model on train set\nxgbr_mod.fit(X_train, y_train)\n\n# Predict on test set\ny_pred = xgbr_mod.predict(X_test)\n\nprint(f\"- {xgbr_mod.__class__.__name__}\")\nprint(f\"R²: {rsqr_score(y_test, y_pred)}\")\nprint(f\"RMSE: {rmse_score(y_test, y_pred)}\")","metadata":{"id":"9bd8d71b","outputId":"b193ba4e-89f2-407f-ea4a-3b485dc7153d","execution":{"iopub.status.busy":"2022-07-05T03:34:10.574796Z","iopub.execute_input":"2022-07-05T03:34:10.575166Z","iopub.status.idle":"2022-07-05T03:34:11.674824Z","shell.execute_reply.started":"2022-07-05T03:34:10.575137Z","shell.execute_reply":"2022-07-05T03:34:11.673939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model results into lists\nmodel_list.append(xgbr_mod.__class__.__name__)\nr2_list.append(round(rsqr_score(y_test, y_pred), 4))\nrmse_list.append(round(rmse_score(y_test, y_pred), 4))","metadata":{"id":"aaac8718","execution":{"iopub.status.busy":"2022-07-05T03:34:11.678064Z","iopub.execute_input":"2022-07-05T03:34:11.678762Z","iopub.status.idle":"2022-07-05T03:34:11.692979Z","shell.execute_reply.started":"2022-07-05T03:34:11.678724Z","shell.execute_reply":"2022-07-05T03:34:11.691245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Actual vs. Predicted house prices\nactual_price = np.exp(y_test[\"SalePriceLog\"])\npredicted_price = np.exp(y_pred)\n\nplt.figure()\nplt.title(\"Actual vs. Predicted house prices\\n (XGBoost)\", fontsize=20)\nplt.scatter(actual_price, predicted_price,\n color=\"deepskyblue\", marker=\"o\", facecolors=\"none\")\nplt.plot([0, 800000], [0, 800000], \"darkorange\", lw=2)\nplt.xlim(0, 800000)\nplt.ylim(0, 800000)\nplt.xlabel(\"\\nActual Price\", fontsize=16)\nplt.ylabel(\"Predicted Price\\n\", fontsize=16)\nplt.show()","metadata":{"id":"95ca54ba","outputId":"11abcbbb-a926-4b0a-bfe0-0915355b6a91","execution":{"iopub.status.busy":"2022-07-05T03:34:11.696039Z","iopub.execute_input":"2022-07-05T03:34:11.697267Z","iopub.status.idle":"2022-07-05T03:34:11.962763Z","shell.execute_reply.started":"2022-07-05T03:34:11.697167Z","shell.execute_reply":"2022-07-05T03:34:11.961590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **IV.2.4. LightGBM regression**","metadata":{"id":"132adcca"}},{"cell_type":"markdown","source":"**LightGBM is also one of the most popular algorithms that are based on Gradient Boosted Machines.**\n\n**[Click here for more information about LightGBM](https://neptune.ai/blog/xgboost-vs-lightgbm)**","metadata":{"id":"3cfa16f1"}},{"cell_type":"code","source":"# Define hyperparameters\ntuned_parameters = {\"max_depth\": [3, 6, 10], \"learning_rate\": [\n 0.01, 0.05, 0.1], \"n_estimators\": [100, 500, 1000], }\n\n# GridSearch\nlgbm_cv = GridSearchCV(estimator=LGBMRegressor(\n), param_grid=tuned_parameters, cv=10, n_jobs=-1, verbose=1)\n\n# fit the GridSearch on train set\nlgbm_cv.fit(X_train, y_train)\n\n# print best params and the corresponding R²\nprint(f\"Best hyperparameters: {lgbm_cv.best_params_}\\n\")\nprint(f\"Best R²: {lgbm_cv.best_score_}\")","metadata":{"id":"2fb6b726","outputId":"d50f7154-06c9-49d8-f3c1-fa6158e455bb","execution":{"iopub.status.busy":"2022-07-05T03:34:11.964289Z","iopub.execute_input":"2022-07-05T03:34:11.964571Z","iopub.status.idle":"2022-07-05T03:34:41.408803Z","shell.execute_reply.started":"2022-07-05T03:34:11.964542Z","shell.execute_reply":"2022-07-05T03:34:41.408120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LGBM Regressor with the best hyperparameters\nlgbm_mod = LGBMRegressor(learning_rate=lgbm_cv.best_params_[\"learning_rate\"],\n max_depth=lgbm_cv.best_params_[\"max_depth\"],\n n_estimators=lgbm_cv.best_params_[\"n_estimators\"])\n\n# Fit the model on train set\nlgbm_mod.fit(X_train, y_train)\n\n# Predict on test set\ny_pred = lgbm_mod.predict(X_test)\n\nprint(f\"- {lgbm_mod.__class__.__name__}\")\nprint(f\"R²: {rsqr_score(y_test, y_pred)}\")\nprint(f\"RMSE: {rmse_score(y_test, y_pred)}\")","metadata":{"id":"106a4088","outputId":"ccf5b252-2eff-4fe4-eb51-29df494a3909","execution":{"iopub.status.busy":"2022-07-05T03:34:41.409982Z","iopub.execute_input":"2022-07-05T03:34:41.410785Z","iopub.status.idle":"2022-07-05T03:34:41.573731Z","shell.execute_reply.started":"2022-07-05T03:34:41.410750Z","shell.execute_reply":"2022-07-05T03:34:41.572940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the model results into lists\nmodel_list.append(lgbm_mod.__class__.__name__)\nr2_list.append(round(rsqr_score(y_test, y_pred), 4))\nrmse_list.append(round(rmse_score(y_test, y_pred), 4))","metadata":{"id":"071dc304","execution":{"iopub.status.busy":"2022-07-05T03:34:41.575165Z","iopub.execute_input":"2022-07-05T03:34:41.575718Z","iopub.status.idle":"2022-07-05T03:34:41.584209Z","shell.execute_reply.started":"2022-07-05T03:34:41.575684Z","shell.execute_reply":"2022-07-05T03:34:41.583418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Actual vs. Predicted house prices\nactual_price = np.exp(y_test[\"SalePriceLog\"])\npredicted_price = np.exp(y_pred)\n\nplt.figure()\nplt.title(\"Actual vs. Predicted house prices\\n (LGBM)\", fontsize=20)\nplt.scatter(actual_price, predicted_price,\n color=\"deepskyblue\", marker=\"o\", facecolors=\"none\")\nplt.plot([0, 800000], [0, 800000], \"darkorange\", lw=2)\nplt.xlim(0, 800000)\nplt.ylim(0, 800000)\nplt.xlabel(\"\\nActual Price\", fontsize=16)\nplt.ylabel(\"Predicted Price\\n\", fontsize=16)\nplt.show()","metadata":{"id":"3cf22119","outputId":"fae0857d-507c-4755-fdf3-d085c1cdb341","execution":{"iopub.status.busy":"2022-07-05T03:34:41.585624Z","iopub.execute_input":"2022-07-05T03:34:41.586081Z","iopub.status.idle":"2022-07-05T03:34:41.820878Z","shell.execute_reply.started":"2022-07-05T03:34:41.586052Z","shell.execute_reply":"2022-07-05T03:34:41.819484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **IV.3. Choosing the best model**","metadata":{"id":"05645004"}},{"cell_type":"code","source":"# Create a table with pd.DataFrame\nmodel_results = pd.DataFrame({\"Model\": model_list,\n \"R²\": r2_list,\n \"RMSE\": rmse_list})\n\nmodel_results","metadata":{"id":"1c8b35f1","outputId":"f733eb11-8aa6-42a4-a2e5-ea506626e1f4","execution":{"iopub.status.busy":"2022-07-05T03:34:41.823231Z","iopub.execute_input":"2022-07-05T03:34:41.824317Z","iopub.status.idle":"2022-07-05T03:34:41.837851Z","shell.execute_reply.started":"2022-07-05T03:34:41.824273Z","shell.execute_reply":"2022-07-05T03:34:41.836657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The results are the best performances in terms of R squared (R²) and Root Mean Square Error (RMSE) correspond to Ridge, Lasso , LGMRegressor and XGB Regressor.**\n \n**Based on these results, we can conclude that the XGB Regressor model gives us the best performance.**\n \n**Thus, XGB Regressor model will be chosen to predict house prices of the Test set of this Kaggle competition.**","metadata":{"id":"766801e4"}},{"cell_type":"markdown","source":"## **IV.4. Prediction on 'House Prices-Advanced Regression Techniques' test data set**","metadata":{"id":"09c1a7ff"}},{"cell_type":"markdown","source":"**By spliting the training data into X_train and X_test we kind of lost some of this training data. A solution will be to update our model chosen on the X_test dataset before predicting on the test dataset of the competition.**","metadata":{"id":"649a3e30"}},{"cell_type":"code","source":"# Predictions from Ridge model\npredictions_list = xgbr_mod.predict(df_test_new)\n\n# Conversion of logarithmic predictions to logical data Sale Price\nsaleprice_preds = np.exp(predictions_list)\n\n# DataFrame of test ID and their corresponding predictions\noutput = pd.DataFrame({\"Id\": Id_test_list,\n \"SalePrice\": saleprice_preds})\noutput.head(10)","metadata":{"id":"9683299e","outputId":"1df4abff-2beb-4074-cda4-4e51ec6849a5","execution":{"iopub.status.busy":"2022-07-05T03:34:41.839601Z","iopub.execute_input":"2022-07-05T03:34:41.841059Z","iopub.status.idle":"2022-07-05T03:34:41.868558Z","shell.execute_reply.started":"2022-07-05T03:34:41.840933Z","shell.execute_reply":"2022-07-05T03:34:41.864427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the output\noutput.to_csv(\"submission.csv\", index=False)","metadata":{"id":"f2596a3e","execution":{"iopub.status.busy":"2022-07-05T03:34:41.869920Z","iopub.execute_input":"2022-07-05T03:34:41.870428Z","iopub.status.idle":"2022-07-05T03:34:41.879913Z","shell.execute_reply.started":"2022-07-05T03:34:41.870395Z","shell.execute_reply":"2022-07-05T03:34:41.878559Z"},"trusted":true},"execution_count":null,"outputs":[]}]}