{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import skew\nfrom sklearn.preprocessing import PowerTransformer, QuantileTransformer, OneHotEncoder, MinMaxScaler\nfrom category_encoders.hashing import HashingEncoder\nfrom sklearn.linear_model import LinearRegression, Lasso, Ridge\nfrom sklearn.model_selection import cross_val_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-14T20:16:13.919932Z","iopub.execute_input":"2022-07-14T20:16:13.920553Z","iopub.status.idle":"2022-07-14T20:16:13.935953Z","shell.execute_reply.started":"2022-07-14T20:16:13.920492Z","shell.execute_reply":"2022-07-14T20:16:13.934201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ntest =  pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:13.943841Z","iopub.execute_input":"2022-07-14T20:16:13.944406Z","iopub.status.idle":"2022-07-14T20:16:14.033389Z","shell.execute_reply.started":"2022-07-14T20:16:13.944357Z","shell.execute_reply":"2022-07-14T20:16:14.032202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.035085Z","iopub.execute_input":"2022-07-14T20:16:14.035696Z","iopub.status.idle":"2022-07-14T20:16:14.046092Z","shell.execute_reply.started":"2022-07-14T20:16:14.035629Z","shell.execute_reply":"2022-07-14T20:16:14.044542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.048037Z","iopub.execute_input":"2022-07-14T20:16:14.048881Z","iopub.status.idle":"2022-07-14T20:16:14.077800Z","shell.execute_reply.started":"2022-07-14T20:16:14.048832Z","shell.execute_reply":"2022-07-14T20:16:14.076275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.078947Z","iopub.execute_input":"2022-07-14T20:16:14.079684Z","iopub.status.idle":"2022-07-14T20:16:14.105487Z","shell.execute_reply.started":"2022-07-14T20:16:14.079650Z","shell.execute_reply":"2022-07-14T20:16:14.104242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get variables\nprint(train.columns.tolist())\n#missing values\nprint(train.isnull().sum().sort_values(ascending = False))\n#percentage\nprint(train.isnull().mean().sort_values(ascending = False))   ","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.107447Z","iopub.execute_input":"2022-07-14T20:16:14.108381Z","iopub.status.idle":"2022-07-14T20:16:14.130091Z","shell.execute_reply.started":"2022-07-14T20:16:14.108346Z","shell.execute_reply":"2022-07-14T20:16:14.128768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"column PoolQC, MiscFeature, Alley, Fence has more than 80% missing values\n\n## Discuss SalePrice column (target)","metadata":{}},{"cell_type":"code","source":"sns.boxplot(train.SalePrice)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.132029Z","iopub.execute_input":"2022-07-14T20:16:14.132845Z","iopub.status.idle":"2022-07-14T20:16:14.320441Z","shell.execute_reply.started":"2022-07-14T20:16:14.132798Z","shell.execute_reply":"2022-07-14T20:16:14.318805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train.corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.321775Z","iopub.execute_input":"2022-07-14T20:16:14.322183Z","iopub.status.idle":"2022-07-14T20:16:14.779515Z","shell.execute_reply.started":"2022-07-14T20:16:14.322148Z","shell.execute_reply":"2022-07-14T20:16:14.778371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corrFilter(x: pd.DataFrame, bound: float):\n    xCorr = x.corr()\n    xFiltered = xCorr[((xCorr >= bound) | (xCorr <= -bound)) & (xCorr !=1.000)]\n    xFlattened = xFiltered.unstack().sort_values().drop_duplicates()\n    return xFlattened\n\ncorrFilter(train, .95)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.781338Z","iopub.execute_input":"2022-07-14T20:16:14.783859Z","iopub.status.idle":"2022-07-14T20:16:14.806322Z","shell.execute_reply.started":"2022-07-14T20:16:14.783822Z","shell.execute_reply":"2022-07-14T20:16:14.804989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"no pairs of correlation greater than 0.95","metadata":{}},{"cell_type":"code","source":"# check skewness for target\nskew(train.SalePrice)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.807745Z","iopub.execute_input":"2022-07-14T20:16:14.808177Z","iopub.status.idle":"2022-07-14T20:16:14.815941Z","shell.execute_reply.started":"2022-07-14T20:16:14.808143Z","shell.execute_reply":"2022-07-14T20:16:14.814854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check skewness for all columns\ntrain_numeric = train.select_dtypes(exclude=['object']).dropna()\nskewness = train_numeric.apply(lambda x: skew(x), axis = 0)\n\n# columns need to change skewness\nskw_cols = skewness[(skewness>4) | (skewness<-4)].index.tolist()\nskw_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.817524Z","iopub.execute_input":"2022-07-14T20:16:14.818059Z","iopub.status.idle":"2022-07-14T20:16:14.843515Z","shell.execute_reply.started":"2022-07-14T20:16:14.818023Z","shell.execute_reply":"2022-07-14T20:16:14.842298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical number of unique values\ntrain.nunique().sort_values(ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.845354Z","iopub.execute_input":"2022-07-14T20:16:14.845794Z","iopub.status.idle":"2022-07-14T20:16:14.868139Z","shell.execute_reply.started":"2022-07-14T20:16:14.845761Z","shell.execute_reply":"2022-07-14T20:16:14.867317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete columns with too many unique values\ncount_unique = train.nunique().sort_values(ascending = False)\ndel_col_cat = count_unique[count_unique > len(train)/2].index.tolist()\ndel_col_cat","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.872868Z","iopub.execute_input":"2022-07-14T20:16:14.873932Z","iopub.status.idle":"2022-07-14T20:16:14.897558Z","shell.execute_reply.started":"2022-07-14T20:16:14.873877Z","shell.execute_reply":"2022-07-14T20:16:14.896431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data cleaning","metadata":{}},{"cell_type":"code","source":"del_name = train.isnull().mean().sort_values(ascending = False)[:4].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.899141Z","iopub.execute_input":"2022-07-14T20:16:14.899507Z","iopub.status.idle":"2022-07-14T20:16:14.912160Z","shell.execute_reply.started":"2022-07-14T20:16:14.899474Z","shell.execute_reply":"2022-07-14T20:16:14.910824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop missing values > 80%\ntrain.drop(del_name, axis = 1, inplace = True)\ntest.drop(del_name, axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.914127Z","iopub.execute_input":"2022-07-14T20:16:14.914998Z","iopub.status.idle":"2022-07-14T20:16:14.927936Z","shell.execute_reply.started":"2022-07-14T20:16:14.914958Z","shell.execute_reply":"2022-07-14T20:16:14.926349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop cols with too many unique values\ntrain.drop(del_col_cat, axis = 1, inplace = True)\ntest.drop(del_col_cat, axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.929532Z","iopub.execute_input":"2022-07-14T20:16:14.930707Z","iopub.status.idle":"2022-07-14T20:16:14.940870Z","shell.execute_reply.started":"2022-07-14T20:16:14.930652Z","shell.execute_reply":"2022-07-14T20:16:14.939786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num = train.select_dtypes(exclude=['object']).apply(lambda x: x.fillna(x.mean()), axis = 1)\ntrain_cat = train.select_dtypes(include=['object']).apply(lambda x: x.fillna(x.mode()[0]), axis = 1)\ntrain = pd.concat([train_num, train_cat], axis = 1)\n\ntest_num = test.select_dtypes(exclude=['object']).apply(lambda x: x.fillna(x.mean()), axis = 1)\ntest_cat = test.select_dtypes(include=['object']).apply(lambda x: x.fillna(x.mode()[0]), axis = 1)\ntest = pd.concat([test_num, test_cat], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:14.942101Z","iopub.execute_input":"2022-07-14T20:16:14.942776Z","iopub.status.idle":"2022-07-14T20:16:17.213701Z","shell.execute_reply.started":"2022-07-14T20:16:14.942742Z","shell.execute_reply":"2022-07-14T20:16:17.212509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# feature engineering\n\ncategorical columns: hash encoder and one hot encoder, consider number of unique values\n\nnumerical columns: robust scaler and PowerTransformer","metadata":{}},{"cell_type":"code","source":"train_count_cat = train.select_dtypes(include=['object']).nunique().sort_values(ascending = False)\ncols_hash = train_count_cat[train_count_cat>3].index.tolist()\ncols_onehot = train_count_cat[train_count_cat<=3].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:17.215286Z","iopub.execute_input":"2022-07-14T20:16:17.216103Z","iopub.status.idle":"2022-07-14T20:16:17.236930Z","shell.execute_reply.started":"2022-07-14T20:16:17.216050Z","shell.execute_reply":"2022-07-14T20:16:17.235709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hash_encoder = HashingEncoder()\nhash_encoder.fit(train[cols_hash])\ntrain_hash = hash_encoder.transform(train[cols_hash])\ntest_hash = hash_encoder.transform(test[cols_hash])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:17.238575Z","iopub.execute_input":"2022-07-14T20:16:17.239293Z","iopub.status.idle":"2022-07-14T20:16:18.690622Z","shell.execute_reply.started":"2022-07-14T20:16:17.239254Z","shell.execute_reply":"2022-07-14T20:16:18.689183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onehot_encoder = OneHotEncoder(handle_unknown = 'ignore')\nonehot_encoder.fit(train[cols_onehot])\ntrain_onehot = onehot_encoder.transform(train[cols_onehot]).toarray()\ntest_onehot = onehot_encoder.transform(test[cols_onehot]).toarray()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.693059Z","iopub.execute_input":"2022-07-14T20:16:18.693538Z","iopub.status.idle":"2022-07-14T20:16:18.712686Z","shell.execute_reply.started":"2022-07-14T20:16:18.693485Z","shell.execute_reply":"2022-07-14T20:16:18.711625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check skewness for all columns\ntrain_numeric = train.select_dtypes(exclude=['object']).dropna()\nskewness = train_numeric.apply(lambda x: skew(x), axis = 0)\n\n# columns need to change skewness\nskw_cols = skewness[(skewness>3) | (skewness<-3)].index.tolist()\nskw_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.714538Z","iopub.execute_input":"2022-07-14T20:16:18.715337Z","iopub.status.idle":"2022-07-14T20:16:18.737361Z","shell.execute_reply.started":"2022-07-14T20:16:18.715292Z","shell.execute_reply":"2022-07-14T20:16:18.736189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power_transform = PowerTransformer()\npower_transform.fit(train[skw_cols])\ntrain_skew = power_transform.transform(train[skw_cols])\ntest_skew = power_transform.transform(test[skw_cols])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.739866Z","iopub.execute_input":"2022-07-14T20:16:18.740684Z","iopub.status.idle":"2022-07-14T20:16:18.790249Z","shell.execute_reply.started":"2022-07-14T20:16:18.740601Z","shell.execute_reply":"2022-07-14T20:16:18.789051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(cols_hash+cols_onehot+skw_cols, axis = 1, inplace = True)\ntest.drop(cols_hash+cols_onehot+skw_cols, axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.791915Z","iopub.execute_input":"2022-07-14T20:16:18.792603Z","iopub.status.idle":"2022-07-14T20:16:18.801816Z","shell.execute_reply.started":"2022-07-14T20:16:18.792557Z","shell.execute_reply":"2022-07-14T20:16:18.800705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train, pd.DataFrame(train_hash), pd.DataFrame(train_onehot), pd.DataFrame(train_skew)], axis = 1)\ntest = pd.concat([test, pd.DataFrame(test_hash), pd.DataFrame(test_onehot), pd.DataFrame(test_skew)], axis = 1)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.803452Z","iopub.execute_input":"2022-07-14T20:16:18.804158Z","iopub.status.idle":"2022-07-14T20:16:18.845125Z","shell.execute_reply.started":"2022-07-14T20:16:18.804111Z","shell.execute_reply":"2022-07-14T20:16:18.843735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.846347Z","iopub.execute_input":"2022-07-14T20:16:18.846692Z","iopub.status.idle":"2022-07-14T20:16:18.864170Z","shell.execute_reply.started":"2022-07-14T20:16:18.846632Z","shell.execute_reply":"2022-07-14T20:16:18.862915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train.drop(['SalePrice'], axis = 1); train_y = train.SalePrice\ntest_X = test","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.865498Z","iopub.execute_input":"2022-07-14T20:16:18.865844Z","iopub.status.idle":"2022-07-14T20:16:18.878392Z","shell.execute_reply.started":"2022-07-14T20:16:18.865814Z","shell.execute_reply":"2022-07-14T20:16:18.877488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min max scaler\nmin_max_scale = MinMaxScaler()\nmin_max_scale.fit(train_X)\ntrain_X = min_max_scale.transform(train_X)\ntest_X = min_max_scale.transform(test_X)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.879985Z","iopub.execute_input":"2022-07-14T20:16:18.880729Z","iopub.status.idle":"2022-07-14T20:16:18.899518Z","shell.execute_reply.started":"2022-07-14T20:16:18.880684Z","shell.execute_reply":"2022-07-14T20:16:18.898269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(train_X).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.901409Z","iopub.execute_input":"2022-07-14T20:16:18.901871Z","iopub.status.idle":"2022-07-14T20:16:18.928496Z","shell.execute_reply.started":"2022-07-14T20:16:18.901824Z","shell.execute_reply":"2022-07-14T20:16:18.927358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sklearn\nsorted(sklearn.metrics.SCORERS.keys())","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.930030Z","iopub.execute_input":"2022-07-14T20:16:18.930466Z","iopub.status.idle":"2022-07-14T20:16:18.937844Z","shell.execute_reply.started":"2022-07-14T20:16:18.930432Z","shell.execute_reply":"2022-07-14T20:16:18.937086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# linear regression\nlr = LinearRegression()\nscores = -sum(cross_val_score(lr, pd.DataFrame(train_X), train_y, cv=10, scoring='neg_root_mean_squared_error'))/10\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:16:18.939169Z","iopub.execute_input":"2022-07-14T20:16:18.939452Z","iopub.status.idle":"2022-07-14T20:16:19.084715Z","shell.execute_reply.started":"2022-07-14T20:16:18.939427Z","shell.execute_reply":"2022-07-14T20:16:19.083434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lasso regression\nlasso = Lasso()\nscores = -sum(cross_val_score(lasso, pd.DataFrame(train_X), train_y, cv=10, scoring='neg_root_mean_squared_error'))/10\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:17:09.173624Z","iopub.execute_input":"2022-07-14T20:17:09.174022Z","iopub.status.idle":"2022-07-14T20:17:09.508792Z","shell.execute_reply.started":"2022-07-14T20:17:09.173992Z","shell.execute_reply":"2022-07-14T20:17:09.507688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ridge = Ridge()\nscores = -sum(cross_val_score(ridge, pd.DataFrame(train_X), train_y, cv=10, scoring='neg_root_mean_squared_error'))/10\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:17:31.852232Z","iopub.execute_input":"2022-07-14T20:17:31.852756Z","iopub.status.idle":"2022-07-14T20:17:31.983096Z","shell.execute_reply.started":"2022-07-14T20:17:31.852724Z","shell.execute_reply":"2022-07-14T20:17:31.981885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission\nridge = Ridge()\nridge.fit(pd.DataFrame(train_X), train_y)\ny_pred = ridge.predict(pd.DataFrame(test_X))\nsubmit = pd.DataFrame()\nsubmit['Id'] = list(range(1461,2920))\nsubmit['SalePrice'] = y_pred\nsubmit.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T20:35:48.544591Z","iopub.execute_input":"2022-07-14T20:35:48.544999Z","iopub.status.idle":"2022-07-14T20:35:48.572960Z","shell.execute_reply.started":"2022-07-14T20:35:48.544965Z","shell.execute_reply":"2022-07-14T20:35:48.571600Z"},"trusted":true},"execution_count":null,"outputs":[]}]}