{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# data analysis and wrangling\nimport pandas as pd\nimport numpy as np\n\n# data visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nplt.style.use('seaborn')\nsns.set(font_scale=2.5)\n\n# find NULL Data\nimport missingno as msno\n\n# ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nfrom scipy.stats import norm, skew ","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.054939Z","iopub.execute_input":"2022-07-11T08:27:18.055470Z","iopub.status.idle":"2022-07-11T08:27:18.072542Z","shell.execute_reply.started":"2022-07-11T08:27:18.055404Z","shell.execute_reply":"2022-07-11T08:27:18.069782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset\ndf_train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ndf_test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\nids = df_test['Id'].values","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.074510Z","iopub.execute_input":"2022-07-11T08:27:18.075461Z","iopub.status.idle":"2022-07-11T08:27:18.128329Z","shell.execute_reply.started":"2022-07-11T08:27:18.075402Z","shell.execute_reply":"2022-07-11T08:27:18.127207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.129596Z","iopub.execute_input":"2022-07-11T08:27:18.130099Z","iopub.status.idle":"2022-07-11T08:27:18.155564Z","shell.execute_reply.started":"2022-07-11T08:27:18.130068Z","shell.execute_reply":"2022-07-11T08:27:18.154350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NaN data\nfor col in df_train.columns:\n    msg = 'columns: {:>10}\\t Percent of NaN value: {:.2f}%'.format(col, 100 * (df_train[col].isnull().sum() / df_train[col].shape[0]))\n    print(msg)\nprint(\"_\"*50 + \"\\n\")\nfor col in df_test.columns:\n    msg = 'columns: {:>10}\\t Percent of NaN value: {:.2f}%'.format(col, 100 * (df_test[col].isnull().sum() / df_test[col].shape[0]))\n    print(msg)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.159115Z","iopub.execute_input":"2022-07-11T08:27:18.159593Z","iopub.status.idle":"2022-07-11T08:27:18.244125Z","shell.execute_reply.started":"2022-07-11T08:27:18.159535Z","shell.execute_reply":"2022-07-11T08:27:18.242996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(df=df_train.iloc[:, :], figsize=(20, 8), color=(0.1, 0.6, 0.8))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.245816Z","iopub.execute_input":"2022-07-11T08:27:18.246501Z","iopub.status.idle":"2022-07-11T08:27:18.837258Z","shell.execute_reply.started":"2022-07-11T08:27:18.246457Z","shell.execute_reply":"2022-07-11T08:27:18.836417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(df=df_test.iloc[:, :], figsize=(20, 8), color=(0.1, 0.6, 0.8))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:18.838314Z","iopub.execute_input":"2022-07-11T08:27:18.839346Z","iopub.status.idle":"2022-07-11T08:27:19.270770Z","shell.execute_reply.started":"2022-07-11T08:27:18.839309Z","shell.execute_reply":"2022-07-11T08:27:19.269624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.272224Z","iopub.execute_input":"2022-07-11T08:27:19.272931Z","iopub.status.idle":"2022-07-11T08:27:19.384171Z","shell.execute_reply.started":"2022-07-11T08:27:19.272889Z","shell.execute_reply":"2022-07-11T08:27:19.383064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.385857Z","iopub.execute_input":"2022-07-11T08:27:19.386550Z","iopub.status.idle":"2022-07-11T08:27:19.395017Z","shell.execute_reply.started":"2022-07-11T08:27:19.386505Z","shell.execute_reply":"2022-07-11T08:27:19.393819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(df_train['SalePrice'])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.400530Z","iopub.execute_input":"2022-07-11T08:27:19.401013Z","iopub.status.idle":"2022-07-11T08:27:19.835152Z","shell.execute_reply.started":"2022-07-11T08:27:19.400969Z","shell.execute_reply":"2022-07-11T08:27:19.833956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = df_train.drop('SalePrice', 1)\ny_train = df_train.SalePrice.values","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.836873Z","iopub.execute_input":"2022-07-11T08:27:19.837675Z","iopub.status.idle":"2022-07-11T08:27:19.845931Z","shell.execute_reply.started":"2022-07-11T08:27:19.837627Z","shell.execute_reply":"2022-07-11T08:27:19.844618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.drop(\"Id\", axis = 1, inplace = True)\ndf_test.drop(\"Id\", axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.847607Z","iopub.execute_input":"2022-07-11T08:27:19.848073Z","iopub.status.idle":"2022-07-11T08:27:19.861920Z","shell.execute_reply.started":"2022-07-11T08:27:19.848029Z","shell.execute_reply":"2022-07-11T08:27:19.860934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.scatter(x = df_train['GrLivArea'], y = df_train['SalePrice'])\nplt.ylabel('SalePrice', fontsize=13)\nplt.xlabel('GrLivArea', fontsize=13)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:19.863195Z","iopub.execute_input":"2022-07-11T08:27:19.868240Z","iopub.status.idle":"2022-07-11T08:27:20.101067Z","shell.execute_reply.started":"2022-07-11T08:27:19.868195Z","shell.execute_reply":"2022-07-11T08:27:20.098875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.drop(df_train[(df_train['GrLivArea']>4000) & (df_train['SalePrice']<300000)].index)\n\n#Check the graphic again\nfig, ax = plt.subplots()\nax.scatter(df_train['GrLivArea'], df_train['SalePrice'])\nplt.ylabel('SalePrice', fontsize=13)\nplt.xlabel('GrLivArea', fontsize=13)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:20.102730Z","iopub.execute_input":"2022-07-11T08:27:20.106407Z","iopub.status.idle":"2022-07-11T08:27:20.391339Z","shell.execute_reply.started":"2022-07-11T08:27:20.106370Z","shell.execute_reply":"2022-07-11T08:27:20.390222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = sns.distplot(df_train['SalePrice'] , fit=norm)\nres.set_title(\"SalePrice distribution\")\n\n# qq plot\nfig = plt.figure()\nres = stats.probplot(df_train['SalePrice'], plot=plt)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:20.392814Z","iopub.execute_input":"2022-07-11T08:27:20.393909Z","iopub.status.idle":"2022-07-11T08:27:21.025558Z","shell.execute_reply.started":"2022-07-11T08:27:20.393863Z","shell.execute_reply":"2022-07-11T08:27:21.024262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"SalePrice\"] = np.log1p(df_train[\"SalePrice\"]) #로그 변환\nres = sns.distplot(df_train['SalePrice'] , fit=norm)\nres.set_title(\"SalePrice distributio\")\n\n# qq plot\nfig = plt.figure()\nres = stats.probplot(df_train['SalePrice'], plot=plt)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:21.027458Z","iopub.execute_input":"2022-07-11T08:27:21.028266Z","iopub.status.idle":"2022-07-11T08:27:21.590633Z","shell.execute_reply.started":"2022-07-11T08:27:21.028216Z","shell.execute_reply":"2022-07-11T08:27:21.589524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"# dataset\ndf_train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ndf_test = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\nids = df_test['Id'].values","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:21.592094Z","iopub.execute_input":"2022-07-11T08:27:21.592549Z","iopub.status.idle":"2022-07-11T08:27:21.639274Z","shell.execute_reply.started":"2022-07-11T08:27:21.592506Z","shell.execute_reply":"2022-07-11T08:27:21.638210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ntrain = df_train.shape[0]\nntest = df_test.shape[0]\ny_train = df_train.SalePrice.values\nall_data = pd.concat((df_train, df_test)).reset_index(drop=True)\nall_data.drop(['SalePrice'], axis=1, inplace=True)\nprint(\"all_data size is : {}\".format(all_data.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:21.641191Z","iopub.execute_input":"2022-07-11T08:27:21.641625Z","iopub.status.idle":"2022-07-11T08:27:21.670836Z","shell.execute_reply.started":"2022-07-11T08:27:21.641583Z","shell.execute_reply":"2022-07-11T08:27:21.669666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)[:30]\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:21.672485Z","iopub.execute_input":"2022-07-11T08:27:21.673162Z","iopub.status.idle":"2022-07-11T08:27:21.705539Z","shell.execute_reply.started":"2022-07-11T08:27:21.673120Z","shell.execute_reply":"2022-07-11T08:27:21.704457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(13, 10))\nplt.xticks(rotation='90')\nsns.barplot(x=all_data_na.index, y=all_data_na)\nplt.xlabel('Features', fontsize=15)\nplt.ylabel('Percent of missing values', fontsize=15)\nplt.ylabel('Percent of missing data by feature', fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:21.709806Z","iopub.execute_input":"2022-07-11T08:27:21.710603Z","iopub.status.idle":"2022-07-11T08:27:22.264550Z","shell.execute_reply.started":"2022-07-11T08:27:21.710553Z","shell.execute_reply":"2022-07-11T08:27:22.263341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int_data = []\nfor i in df_train.columns:\n    if type(df_train[i][0]) == type(df_train['SalePrice'][0]):\n        int_data.append(i)\n        \nheatmap_data = df_train[int_data]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:22.265842Z","iopub.execute_input":"2022-07-11T08:27:22.266289Z","iopub.status.idle":"2022-07-11T08:27:22.282186Z","shell.execute_reply.started":"2022-07-11T08:27:22.266234Z","shell.execute_reply":"2022-07-11T08:27:22.280961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heatmap_d1 = heatmap_data.iloc[:, :len(int_data)//2]\nheatmap_d1['SalePrice'] = df_train['SalePrice']\nheatmap_d2 = heatmap_data.iloc[:, len(int_data)//2:]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:22.283707Z","iopub.execute_input":"2022-07-11T08:27:22.284261Z","iopub.status.idle":"2022-07-11T08:27:22.295546Z","shell.execute_reply.started":"2022-07-11T08:27:22.284229Z","shell.execute_reply":"2022-07-11T08:27:22.294668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colormap = plt.cm.Blues\nplt.figure(figsize=(14, 10))\nplt.title('Pearson Correalation of Features', y=1.05, size=15)\nsns.heatmap(heatmap_d1.astype(float).corr(), linewidths=0.1, vmax=1.0,\n           square=True, cmap=colormap, linecolor='white', annot=True, annot_kws={'size':10}, fmt='.2f')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:22.298271Z","iopub.execute_input":"2022-07-11T08:27:22.298767Z","iopub.status.idle":"2022-07-11T08:27:24.122831Z","shell.execute_reply.started":"2022-07-11T08:27:22.298720Z","shell.execute_reply":"2022-07-11T08:27:24.121485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colormap = plt.cm.Blues\nplt.figure(figsize=(14, 10))\nplt.title('Pearson Correalation of Features', y=1.05, size=15)\nsns.heatmap(heatmap_d2.astype(float).corr(), linewidths=0.1, vmax=1.0,\n           square=True, cmap=colormap, linecolor='white', annot=True, annot_kws={'size':10}, fmt='.2f')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:24.124627Z","iopub.execute_input":"2022-07-11T08:27:24.125061Z","iopub.status.idle":"2022-07-11T08:27:25.721101Z","shell.execute_reply.started":"2022-07-11T08:27:24.125017Z","shell.execute_reply":"2022-07-11T08:27:25.719918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inputing missing values","metadata":{}},{"cell_type":"code","source":"all_data['PoolQC'].fillna(\"None\", inplace=True)\nall_data['PoolQC'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.728710Z","iopub.execute_input":"2022-07-11T08:27:25.729090Z","iopub.status.idle":"2022-07-11T08:27:25.740065Z","shell.execute_reply.started":"2022-07-11T08:27:25.729056Z","shell.execute_reply":"2022-07-11T08:27:25.738532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['MiscFeature'].fillna(\"None\", inplace=True)\nall_data['MiscFeature'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.741993Z","iopub.execute_input":"2022-07-11T08:27:25.742763Z","iopub.status.idle":"2022-07-11T08:27:25.756142Z","shell.execute_reply.started":"2022-07-11T08:27:25.742717Z","shell.execute_reply":"2022-07-11T08:27:25.754900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['Alley'].fillna(\"None\", inplace=True)\nall_data['Alley'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.758667Z","iopub.execute_input":"2022-07-11T08:27:25.759707Z","iopub.status.idle":"2022-07-11T08:27:25.769638Z","shell.execute_reply.started":"2022-07-11T08:27:25.759662Z","shell.execute_reply":"2022-07-11T08:27:25.768328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['Fence'].fillna(\"None\", inplace=True)\nall_data['Fence'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.771627Z","iopub.execute_input":"2022-07-11T08:27:25.772806Z","iopub.status.idle":"2022-07-11T08:27:25.781943Z","shell.execute_reply.started":"2022-07-11T08:27:25.772757Z","shell.execute_reply":"2022-07-11T08:27:25.781127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['FireplaceQu'].fillna(\"None\", inplace=True)\nall_data['FireplaceQu'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.783082Z","iopub.execute_input":"2022-07-11T08:27:25.783629Z","iopub.status.idle":"2022-07-11T08:27:25.792142Z","shell.execute_reply.started":"2022-07-11T08:27:25.783569Z","shell.execute_reply":"2022-07-11T08:27:25.791393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['LotFrontage'] = all_data.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(\n    lambda x: x.fillna(x.median()))\nall_data['LotFrontage'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.793533Z","iopub.execute_input":"2022-07-11T08:27:25.794040Z","iopub.status.idle":"2022-07-11T08:27:25.821887Z","shell.execute_reply.started":"2022-07-11T08:27:25.794009Z","shell.execute_reply":"2022-07-11T08:27:25.820843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']\nall_data[cols] = all_data[cols].fillna('None')\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.823205Z","iopub.execute_input":"2022-07-11T08:27:25.823541Z","iopub.status.idle":"2022-07-11T08:27:25.838866Z","shell.execute_reply.started":"2022-07-11T08:27:25.823512Z","shell.execute_reply":"2022-07-11T08:27:25.837936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data[cols] = all_data[cols].fillna('None')\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.840288Z","iopub.execute_input":"2022-07-11T08:27:25.840606Z","iopub.status.idle":"2022-07-11T08:27:25.856623Z","shell.execute_reply.started":"2022-07-11T08:27:25.840578Z","shell.execute_reply":"2022-07-11T08:27:25.855196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['GarageYrBlt', 'GarageArea', 'GarageCars']\nall_data[cols] = all_data[cols].fillna(0)\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.858403Z","iopub.execute_input":"2022-07-11T08:27:25.858856Z","iopub.status.idle":"2022-07-11T08:27:25.873914Z","shell.execute_reply.started":"2022-07-11T08:27:25.858765Z","shell.execute_reply":"2022-07-11T08:27:25.872766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']\nall_data[cols] = all_data[cols].fillna(0)\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.875851Z","iopub.execute_input":"2022-07-11T08:27:25.876269Z","iopub.status.idle":"2022-07-11T08:27:25.889037Z","shell.execute_reply.started":"2022-07-11T08:27:25.876237Z","shell.execute_reply":"2022-07-11T08:27:25.887843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\nall_data[cols] = all_data[cols].fillna('None')\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.890400Z","iopub.execute_input":"2022-07-11T08:27:25.890766Z","iopub.status.idle":"2022-07-11T08:27:25.908333Z","shell.execute_reply.started":"2022-07-11T08:27:25.890730Z","shell.execute_reply":"2022-07-11T08:27:25.907321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data[\"MasVnrType\"].fillna(\"None\", inplace=True)\nall_data[\"MasVnrArea\"].fillna(0, inplace=True)\nall_data[[\"MasVnrType\", \"MasVnrArea\"]].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.909494Z","iopub.execute_input":"2022-07-11T08:27:25.913057Z","iopub.status.idle":"2022-07-11T08:27:25.924940Z","shell.execute_reply.started":"2022-07-11T08:27:25.913019Z","shell.execute_reply":"2022-07-11T08:27:25.924008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['MSZoning'].fillna(all_data['MSZoning'].mode()[0], inplace=True)\nall_data[\"MSZoning\"].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.926305Z","iopub.execute_input":"2022-07-11T08:27:25.927178Z","iopub.status.idle":"2022-07-11T08:27:25.937836Z","shell.execute_reply.started":"2022-07-11T08:27:25.927141Z","shell.execute_reply":"2022-07-11T08:27:25.937017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.drop(['Utilities'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.939314Z","iopub.execute_input":"2022-07-11T08:27:25.940162Z","iopub.status.idle":"2022-07-11T08:27:25.948841Z","shell.execute_reply.started":"2022-07-11T08:27:25.940125Z","shell.execute_reply":"2022-07-11T08:27:25.947725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data[\"Functional\"].fillna(\"Typ\", inplace=True)\nall_data[\"Functional\"].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.950206Z","iopub.execute_input":"2022-07-11T08:27:25.950730Z","iopub.status.idle":"2022-07-11T08:27:25.961783Z","shell.execute_reply.started":"2022-07-11T08:27:25.950693Z","shell.execute_reply":"2022-07-11T08:27:25.960724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['Electrical'].fillna(all_data['Electrical'].mode()[0], inplace=True)\nall_data['Electrical'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.963292Z","iopub.execute_input":"2022-07-11T08:27:25.963779Z","iopub.status.idle":"2022-07-11T08:27:25.977503Z","shell.execute_reply.started":"2022-07-11T08:27:25.963739Z","shell.execute_reply":"2022-07-11T08:27:25.976486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode()[0], inplace=True)\nall_data['KitchenQual'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.978670Z","iopub.execute_input":"2022-07-11T08:27:25.979423Z","iopub.status.idle":"2022-07-11T08:27:25.994609Z","shell.execute_reply.started":"2022-07-11T08:27:25.979389Z","shell.execute_reply":"2022-07-11T08:27:25.993678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['Exterior1st', 'Exterior2nd']\nall_data['Exterior1st'].fillna(all_data['Exterior1st'].mode()[0], inplace=True)\nall_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode()[0], inplace=True)\nall_data[cols].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:25.995935Z","iopub.execute_input":"2022-07-11T08:27:25.996409Z","iopub.status.idle":"2022-07-11T08:27:26.010612Z","shell.execute_reply.started":"2022-07-11T08:27:25.996380Z","shell.execute_reply":"2022-07-11T08:27:26.009781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['SaleType'].fillna(all_data['SaleType'].mode()[0], inplace=True)\nall_data['SaleType'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.011924Z","iopub.execute_input":"2022-07-11T08:27:26.012514Z","iopub.status.idle":"2022-07-11T08:27:26.021114Z","shell.execute_reply.started":"2022-07-11T08:27:26.012473Z","shell.execute_reply":"2022-07-11T08:27:26.020361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['MSSubClass'].fillna(\"None\", inplace=True)\nall_data['MSSubClass'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.022409Z","iopub.execute_input":"2022-07-11T08:27:26.022911Z","iopub.status.idle":"2022-07-11T08:27:26.031417Z","shell.execute_reply.started":"2022-07-11T08:27:26.022882Z","shell.execute_reply":"2022-07-11T08:27:26.030309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.032908Z","iopub.execute_input":"2022-07-11T08:27:26.034761Z","iopub.status.idle":"2022-07-11T08:27:26.065096Z","shell.execute_reply.started":"2022-07-11T08:27:26.034714Z","shell.execute_reply":"2022-07-11T08:27:26.064167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)\n\nall_data['OverallCond'] = all_data['OverallCond'].astype(str)\n\nall_data['YrSold'] = all_data['YrSold'].astype(str)\nall_data['MoSold'] = all_data['MoSold'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.066925Z","iopub.execute_input":"2022-07-11T08:27:26.067750Z","iopub.status.idle":"2022-07-11T08:27:26.089276Z","shell.execute_reply.started":"2022-07-11T08:27:26.067702Z","shell.execute_reply":"2022-07-11T08:27:26.088135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.091707Z","iopub.execute_input":"2022-07-11T08:27:26.092042Z","iopub.status.idle":"2022-07-11T08:27:26.100991Z","shell.execute_reply.started":"2022-07-11T08:27:26.092011Z","shell.execute_reply":"2022-07-11T08:27:26.099656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.103019Z","iopub.execute_input":"2022-07-11T08:27:26.103961Z","iopub.status.idle":"2022-07-11T08:27:26.109504Z","shell.execute_reply.started":"2022-07-11T08:27:26.103752Z","shell.execute_reply":"2022-07-11T08:27:26.108313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', \n        'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1', \n        'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',\n        'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond', \n        'YrSold', 'MoSold')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.111369Z","iopub.execute_input":"2022-07-11T08:27:26.112157Z","iopub.status.idle":"2022-07-11T08:27:26.121553Z","shell.execute_reply.started":"2022-07-11T08:27:26.112111Z","shell.execute_reply":"2022-07-11T08:27:26.120261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# process columns, apply LabelEncoder to categorical features\nfor c in cols:\n    lbl = LabelEncoder() \n    lbl.fit(list(all_data[c].values)) \n    all_data[c] = lbl.transform(list(all_data[c].values))\n\n# shape        \nprint('Shape all_data: {}'.format(all_data.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.122696Z","iopub.execute_input":"2022-07-11T08:27:26.127732Z","iopub.status.idle":"2022-07-11T08:27:26.291875Z","shell.execute_reply.started":"2022-07-11T08:27:26.127672Z","shell.execute_reply":"2022-07-11T08:27:26.290495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.293529Z","iopub.execute_input":"2022-07-11T08:27:26.293868Z","iopub.status.idle":"2022-07-11T08:27:26.301607Z","shell.execute_reply.started":"2022-07-11T08:27:26.293836Z","shell.execute_reply":"2022-07-11T08:27:26.300658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\nskewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)\nprint(\"\\nSkew in numerical features: \\n\")\nskewness = pd.DataFrame({'Skew' :skewed_feats})\nskewness.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.303034Z","iopub.execute_input":"2022-07-11T08:27:26.303590Z","iopub.status.idle":"2022-07-11T08:27:26.351927Z","shell.execute_reply.started":"2022-07-11T08:27:26.303556Z","shell.execute_reply":"2022-07-11T08:27:26.350791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewness = skewness[abs(skewness) > 0.75]\nprint(\"There are {} skewed numerical features to Box Cox transform\".format(skewness.shape[0]))\n\nfrom scipy.special import boxcox1p\nskewed_features = skewness.index\nlam = 0.15\nfor feat in skewed_features:\n    all_data[feat] = boxcox1p(all_data[feat], lam)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.353269Z","iopub.execute_input":"2022-07-11T08:27:26.353614Z","iopub.status.idle":"2022-07-11T08:27:26.395124Z","shell.execute_reply.started":"2022-07-11T08:27:26.353584Z","shell.execute_reply":"2022-07-11T08:27:26.393786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.get_dummies(all_data)\nprint(all_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.396603Z","iopub.execute_input":"2022-07-11T08:27:26.397504Z","iopub.status.idle":"2022-07-11T08:27:26.431176Z","shell.execute_reply.started":"2022-07-11T08:27:26.397408Z","shell.execute_reply":"2022-07-11T08:27:26.430360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = all_data[:ntrain]\ndf_test = all_data[ntrain:]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.432549Z","iopub.execute_input":"2022-07-11T08:27:26.433107Z","iopub.status.idle":"2022-07-11T08:27:26.439039Z","shell.execute_reply.started":"2022-07-11T08:27:26.433071Z","shell.execute_reply":"2022-07-11T08:27:26.437805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model development","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import ElasticNet, Lasso,  BayesianRidge, LassoLarsIC\nfrom sklearn.ensemble import RandomForestRegressor,  GradientBoostingRegressor\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\nfrom sklearn.metrics import mean_squared_error\nimport xgboost as xgb\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:26.441128Z","iopub.execute_input":"2022-07-11T08:27:26.441992Z","iopub.status.idle":"2022-07-11T08:27:27.908905Z","shell.execute_reply.started":"2022-07-11T08:27:26.441943Z","shell.execute_reply":"2022-07-11T08:27:27.907525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Validation function\nn_folds = 5\n\ndef rmsle_cv(model):\n    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(df_train.values)\n    rmse= np.sqrt(-cross_val_score(model, df_train.values, y_train, scoring=\"neg_mean_squared_error\", cv = kf))\n    return(rmse)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.910214Z","iopub.execute_input":"2022-07-11T08:27:27.910585Z","iopub.status.idle":"2022-07-11T08:27:27.918958Z","shell.execute_reply.started":"2022-07-11T08:27:27.910554Z","shell.execute_reply":"2022-07-11T08:27:27.918025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=1))\nENet = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))\nKRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)\nGBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,\n                                   max_depth=4, max_features='sqrt',\n                                   min_samples_leaf=15, min_samples_split=10, \n                                   loss='huber', random_state =5)\nmodel_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468, \n                             learning_rate=0.05, max_depth=3, \n                             min_child_weight=1.7817, n_estimators=2200,\n                             reg_alpha=0.4640, reg_lambda=0.8571,\n                             subsample=0.5213, silent=1,\n                             random_state =7, nthread = -1)\nmodel_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,\n                              learning_rate=0.05, n_estimators=720,\n                              max_bin = 55, bagging_fraction = 0.8,\n                              bagging_freq = 5, feature_fraction = 0.2319,\n                              feature_fraction_seed=9, bagging_seed=9,\n                              min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.920709Z","iopub.execute_input":"2022-07-11T08:27:27.921612Z","iopub.status.idle":"2022-07-11T08:27:27.934300Z","shell.execute_reply.started":"2022-07-11T08:27:27.921576Z","shell.execute_reply":"2022-07-11T08:27:27.933210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, models):\n        self.models = models\n        \n    # we define clones of the original models to fit the data in\n    def fit(self, X, y):\n        self.models_ = [clone(x) for x in self.models]\n        \n        # Train cloned base models\n        for model in self.models_:\n            model.fit(X, y)\n\n        return self\n    \n    #Now we do the predictions for cloned models and average them\n    def predict(self, X):\n        predictions = np.column_stack([\n            model.predict(X) for model in self.models_\n        ])\n        return np.mean(predictions, axis=1) ","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.935796Z","iopub.execute_input":"2022-07-11T08:27:27.937029Z","iopub.status.idle":"2022-07-11T08:27:27.948692Z","shell.execute_reply.started":"2022-07-11T08:27:27.936988Z","shell.execute_reply":"2022-07-11T08:27:27.947205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmsle(y, y_pred):\n    return np.sqrt(mean_squared_error(y, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.951573Z","iopub.execute_input":"2022-07-11T08:27:27.951934Z","iopub.status.idle":"2022-07-11T08:27:27.969972Z","shell.execute_reply.started":"2022-07-11T08:27:27.951903Z","shell.execute_reply":"2022-07-11T08:27:27.968331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, base_models, meta_model, n_folds=5):\n        self.base_models = base_models\n        self.meta_model = meta_model\n        self.n_folds = n_folds\n\n    def fit(self, X, y):\n        self.base_models_ = [list() for x in self.base_models]\n        self.meta_model_ = clone(self.meta_model)\n        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)\n        \n        # Train cloned base models then create out-of-fold predictions\n        # that are needed to train the cloned meta-model\n        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))\n        for i, model in enumerate(self.base_models):\n            for train_index, holdout_index in kfold.split(X, y):\n                instance = clone(model)\n                self.base_models_[i].append(instance)\n                instance.fit(X[train_index], y[train_index])\n                y_pred = instance.predict(X[holdout_index])\n                out_of_fold_predictions[holdout_index, i] = y_pred\n                \n        # Now train the cloned  meta-model using the out-of-fold predictions as new feature\n        self.meta_model_.fit(out_of_fold_predictions, y)\n        return self\n\n    #Do the predictions of all base models on the test data and use the averaged predictions as \n    #meta-features for the final prediction which is done by the meta-model\n    def predict(self, X):\n        meta_features = np.column_stack([\n            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)\n            for base_models in self.base_models_ ])\n        return self.meta_model_.predict(meta_features)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.971629Z","iopub.execute_input":"2022-07-11T08:27:27.972112Z","iopub.status.idle":"2022-07-11T08:27:27.995509Z","shell.execute_reply.started":"2022-07-11T08:27:27.972060Z","shell.execute_reply":"2022-07-11T08:27:27.994306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_averaged_models = StackingAveragedModels(base_models = (ENet, GBoost, KRR),\n                                                 meta_model = lasso)\n\nscore = rmsle_cv(stacked_averaged_models)\nprint(\"Stacking Averaged models score: {:.4f} ({:.4f})\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:27:27.997637Z","iopub.execute_input":"2022-07-11T08:27:27.998140Z","iopub.status.idle":"2022-07-11T08:33:30.780909Z","shell.execute_reply.started":"2022-07-11T08:27:27.998091Z","shell.execute_reply":"2022-07-11T08:33:30.777281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_averaged_models.fit(df_train.values, y_train)\nstacked_train_pred = stacked_averaged_models.predict(df_train.values)\nstacked_pred = np.expm1(stacked_averaged_models.predict(df_test.values))\nprint(rmsle(y_train, stacked_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:33:30.782883Z","iopub.execute_input":"2022-07-11T08:33:30.783415Z","iopub.status.idle":"2022-07-11T08:34:52.216834Z","shell.execute_reply.started":"2022-07-11T08:33:30.783368Z","shell.execute_reply":"2022-07-11T08:34:52.215505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_xgb.fit(df_train, y_train)\nxgb_train_pred = model_xgb.predict(df_train)\nxgb_pred = np.expm1(model_xgb.predict(df_test))\nprint(rmsle(y_train, xgb_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:34:52.219024Z","iopub.execute_input":"2022-07-11T08:34:52.219856Z","iopub.status.idle":"2022-07-11T08:35:05.291920Z","shell.execute_reply.started":"2022-07-11T08:34:52.219810Z","shell.execute_reply":"2022-07-11T08:35:05.290886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lgb.fit(df_train, y_train)\nlgb_train_pred = model_lgb.predict(df_train)\nlgb_pred = np.expm1(model_lgb.predict(df_test.values))\nprint(rmsle(y_train, lgb_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:35:05.293587Z","iopub.execute_input":"2022-07-11T08:35:05.294310Z","iopub.status.idle":"2022-07-11T08:35:05.754775Z","shell.execute_reply.started":"2022-07-11T08:35:05.294263Z","shell.execute_reply":"2022-07-11T08:35:05.753538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:35:05.756592Z","iopub.execute_input":"2022-07-11T08:35:05.757417Z","iopub.status.idle":"2022-07-11T08:35:05.763102Z","shell.execute_reply.started":"2022-07-11T08:35:05.757369Z","shell.execute_reply":"2022-07-11T08:35:05.762105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame()\nsub['Id'] = ids\nsub['SalePrice'] = ensemble\nsub.to_csv('./submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T08:35:05.764844Z","iopub.execute_input":"2022-07-11T08:35:05.765757Z","iopub.status.idle":"2022-07-11T08:35:05.783971Z","shell.execute_reply.started":"2022-07-11T08:35:05.765709Z","shell.execute_reply":"2022-07-11T08:35:05.782837Z"},"trusted":true},"execution_count":null,"outputs":[]}]}