{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Linear modling","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport scipy.stats as ss\nimport pandas as pd\nimport missingno as mno\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.metrics import explained_variance_score\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import make_scorer\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import ElasticNet\nfrom sklearn.linear_model import RANSACRegressor\nfrom sklearn.linear_model import HuberRegressor\nfrom sklearn.linear_model import SGDRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T18:01:17.127107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data","metadata":{}},{"cell_type":"code","source":"house_price_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')\nhouse_price_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:30.740828Z","iopub.execute_input":"2022-07-15T18:01:30.741642Z","iopub.status.idle":"2022-07-15T18:01:30.787493Z","shell.execute_reply.started":"2022-07-15T18:01:30.741609Z","shell.execute_reply":"2022-07-15T18:01:30.786388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')\ntest_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:28.665643Z","iopub.execute_input":"2022-07-15T18:01:28.666018Z","iopub.status.idle":"2022-07-15T18:01:28.712048Z","shell.execute_reply.started":"2022-07-15T18:01:28.665988Z","shell.execute_reply":"2022-07-15T18:01:28.711300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_rows=80\nhouse_price_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:34.297844Z","iopub.execute_input":"2022-07-15T18:01:34.298201Z","iopub.status.idle":"2022-07-15T18:01:34.306515Z","shell.execute_reply.started":"2022-07-15T18:01:34.298166Z","shell.execute_reply":"2022-07-15T18:01:34.305717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = house_price_data.pop('SalePrice')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:36.086621Z","iopub.execute_input":"2022-07-15T18:01:36.087183Z","iopub.status.idle":"2022-07-15T18:01:36.092583Z","shell.execute_reply.started":"2022-07-15T18:01:36.087143Z","shell.execute_reply":"2022-07-15T18:01:36.091610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data = pd.concat([house_price_data, test_data], ignore_index=True)\nhouse_price_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:37.817417Z","iopub.execute_input":"2022-07-15T18:01:37.817962Z","iopub.status.idle":"2022-07-15T18:01:37.855711Z","shell.execute_reply.started":"2022-07-15T18:01:37.817932Z","shell.execute_reply":"2022-07-15T18:01:37.854965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning house price dataset\n\ntraining set contains NaN values which need to manage.","metadata":{}},{"cell_type":"markdown","source":"- ## need to get a idea about missing values","metadata":{}},{"cell_type":"code","source":"mno.matrix(house_price_data,\n           figsize=[42,12],\n           labels=house_price_data.columns.to_list())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:41.079380Z","iopub.execute_input":"2022-07-15T18:01:41.079801Z","iopub.status.idle":"2022-07-15T18:01:43.808536Z","shell.execute_reply.started":"2022-07-15T18:01:41.079769Z","shell.execute_reply":"2022-07-15T18:01:43.807720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### filling columns with NaN values entered because there isn't that feature available.","metadata":{}},{"cell_type":"code","source":"cols_no = ['MiscFeature','Fence','PoolQC','GarageCond','GarageQual',\n           'GarageFinish','FireplaceQu','BsmtFinType2','BsmtFinType1',\n           'BsmtExposure','BsmtCond','BsmtQual','Alley']\n\nhouse_price_data[cols_no] = house_price_data[cols_no].fillna('None')\nhouse_price_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:48.621002Z","iopub.execute_input":"2022-07-15T18:01:48.621691Z","iopub.status.idle":"2022-07-15T18:01:48.663293Z","shell.execute_reply.started":"2022-07-15T18:01:48.621652Z","shell.execute_reply":"2022-07-15T18:01:48.661888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mno.matrix(house_price_data,\n           figsize=[42,12],\n           labels=house_price_data.columns.to_list())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:51.515736Z","iopub.execute_input":"2022-07-15T18:01:51.517882Z","iopub.status.idle":"2022-07-15T18:01:54.287919Z","shell.execute_reply.started":"2022-07-15T18:01:51.517839Z","shell.execute_reply":"2022-07-15T18:01:54.286692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### fill remaining NaN values","metadata":{}},{"cell_type":"code","source":"nan_cols = house_price_data.columns[house_price_data.isna().sum()!=0]\nhouse_price_data[nan_cols].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:01:58.095723Z","iopub.execute_input":"2022-07-15T18:01:58.096091Z","iopub.status.idle":"2022-07-15T18:01:58.113733Z","shell.execute_reply.started":"2022-07-15T18:01:58.096062Z","shell.execute_reply":"2022-07-15T18:01:58.112429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[nan_cols].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:05.609632Z","iopub.execute_input":"2022-07-15T18:02:05.610070Z","iopub.status.idle":"2022-07-15T18:02:05.661494Z","shell.execute_reply.started":"2022-07-15T18:02:05.610040Z","shell.execute_reply":"2022-07-15T18:02:05.660386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_1 = house_price_data.groupby(by=['LotShape'])['LotFrontage'].mean().to_dict()\nmap_1","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:07.898966Z","iopub.execute_input":"2022-07-15T18:02:07.899641Z","iopub.status.idle":"2022-07-15T18:02:07.909738Z","shell.execute_reply.started":"2022-07-15T18:02:07.899606Z","shell.execute_reply":"2022-07-15T18:02:07.908845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_2 = house_price_data.groupby(by=['GarageType'])['GarageYrBlt'].mean().to_dict()\nmap_2","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:09.286685Z","iopub.execute_input":"2022-07-15T18:02:09.287086Z","iopub.status.idle":"2022-07-15T18:02:09.295783Z","shell.execute_reply.started":"2022-07-15T18:02:09.287013Z","shell.execute_reply":"2022-07-15T18:02:09.294533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key, value in map_1.items():\n    temp_id = house_price_data['LotFrontage'][(house_price_data['LotFrontage'].isna()) & (house_price_data['LotShape']==key)].index.to_list()\n    house_price_data.loc[temp_id,'LotFrontage']=value","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:12.626520Z","iopub.execute_input":"2022-07-15T18:02:12.627494Z","iopub.status.idle":"2022-07-15T18:02:12.640988Z","shell.execute_reply.started":"2022-07-15T18:02:12.627448Z","shell.execute_reply":"2022-07-15T18:02:12.639821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[['MasVnrType', 'MasVnrArea', \n                  'GarageType', 'Electrical']] = house_price_data[['MasVnrType', 'MasVnrArea', \n                                                                   'GarageType', 'Electrical']].fillna(method='ffill')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:15.516469Z","iopub.execute_input":"2022-07-15T18:02:15.516911Z","iopub.status.idle":"2022-07-15T18:02:15.531763Z","shell.execute_reply.started":"2022-07-15T18:02:15.516879Z","shell.execute_reply":"2022-07-15T18:02:15.530393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key, value in map_2.items():\n    temp_id = house_price_data['GarageYrBlt'][(house_price_data['GarageYrBlt'].isna()) & (house_price_data['GarageType']==key)].index.to_list()\n    house_price_data.loc[temp_id,'GarageYrBlt']=int(value)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:18.815042Z","iopub.execute_input":"2022-07-15T18:02:18.815934Z","iopub.status.idle":"2022-07-15T18:02:18.831554Z","shell.execute_reply.started":"2022-07-15T18:02:18.815891Z","shell.execute_reply":"2022-07-15T18:02:18.830568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data = house_price_data.fillna(method='ffill')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:20.846613Z","iopub.execute_input":"2022-07-15T18:02:20.846974Z","iopub.status.idle":"2022-07-15T18:02:20.862930Z","shell.execute_reply.started":"2022-07-15T18:02:20.846947Z","shell.execute_reply":"2022-07-15T18:02:20.861963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mno.matrix(house_price_data,\n           figsize=[42,12],\n           labels=house_price_data.columns.to_list())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:23.765600Z","iopub.execute_input":"2022-07-15T18:02:23.767041Z","iopub.status.idle":"2022-07-15T18:02:26.296971Z","shell.execute_reply.started":"2022-07-15T18:02:23.766997Z","shell.execute_reply":"2022-07-15T18:02:26.295783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"num_feat = house_price_data.iloc[:1460].select_dtypes(['int','float']).columns\ncat_feat = house_price_data.iloc[:1460].select_dtypes(['object']).columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:33.517165Z","iopub.execute_input":"2022-07-15T18:02:33.517659Z","iopub.status.idle":"2022-07-15T18:02:33.531790Z","shell.execute_reply.started":"2022-07-15T18:02:33.517624Z","shell.execute_reply":"2022-07-15T18:02:33.530603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data.iloc[:1460].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:35.736143Z","iopub.execute_input":"2022-07-15T18:02:35.736588Z","iopub.status.idle":"2022-07-15T18:02:35.841930Z","shell.execute_reply.started":"2022-07-15T18:02:35.736553Z","shell.execute_reply":"2022-07-15T18:02:35.840619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Distribution of features","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=12, ncols=3, figsize=(16,64), dpi=300)\naxes = axes.ravel()\n\nfor i in range(len(axes)):\n    if not house_price_data[num_feat[i]].dtype=='int64':\n        sns.histplot(data=house_price_data.iloc[:1460], x=num_feat[i], ax=axes[i])\n    else:\n        sns.histplot(data=house_price_data.iloc[:1460], x=num_feat[i], discrete=True, ax=axes[i])\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:00:18.277914Z","iopub.execute_input":"2022-07-15T07:00:18.278348Z","iopub.status.idle":"2022-07-15T07:07:43.659774Z","shell.execute_reply.started":"2022-07-15T07:00:18.278315Z","shell.execute_reply":"2022-07-15T07:07:43.658591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Correlation between features","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,20),dpi=300)\nsns.heatmap(house_price_data.iloc[:1460].corr(),\n            annot=True,\n            fmt='.2f',\n            cbar=False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:08:20.632098Z","iopub.execute_input":"2022-07-15T07:08:20.632595Z","iopub.status.idle":"2022-07-15T07:08:27.709519Z","shell.execute_reply.started":"2022-07-15T07:08:20.632564Z","shell.execute_reply":"2022-07-15T07:08:27.708179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### Looking for outliers ","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=12, ncols=3, figsize=(16,64), dpi=300)\naxes = axes.ravel()\n\nfor i in range(len(axes)):\n    sns.boxplot(data=house_price_data.iloc[:1460], x=num_feat[i], ax=axes[i])\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:08:49.510978Z","iopub.execute_input":"2022-07-15T07:08:49.511376Z","iopub.status.idle":"2022-07-15T07:08:56.029094Z","shell.execute_reply.started":"2022-07-15T07:08:49.511342Z","shell.execute_reply":"2022-07-15T07:08:56.027994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engeneering","metadata":{}},{"cell_type":"markdown","source":"feature description:\n\n- **years between original construction and remodel (YearRemodAdd - YearBuilt)**\n- **un-finished basement ratio to total basement area (BsmtUnfSF/TotalBsmtSF)**\n- **basement size to total surface are (TotalBsmtSF/LotArea)**\n- **floor surface ratio (2ndFlrSF/1stFlrSF)**\n- **number of bathrooms in basement (BsmtFullBath + BsmtHalfBath)**\n- **number of bathrooms above grade (FullBath + HalfBath)**\n- **garage size ratio to total surface area (GarageArea/LotArea)**\n- **wood deck surface area ratio to total area (WoodDeckSF/LotArea)**\n- **total porch area (OpenPorchSF + EnclosedPorch + 3SsnPorch + ScreenPorch)**\n- **surface without screen in encloused porch (ScreenPorch/EnclosedPorch)**\n- **house surface area without garage and pool (LotArea - (GarageArea + PoolArea))**\n- **how many years take to sale after build (YrSold - YearBuilt)**","metadata":{}},{"cell_type":"code","source":"house_price_data['add_feat_1'] = house_price_data['YearRemodAdd'] - house_price_data['YearBuilt']\nhouse_price_data['add_feat_2'] = house_price_data['BsmtUnfSF']/house_price_data['TotalBsmtSF']\nhouse_price_data['add_feat_3'] = house_price_data['TotalBsmtSF']/house_price_data['LotArea']\nhouse_price_data['add_feat_4'] = house_price_data['2ndFlrSF']/house_price_data['1stFlrSF']\nhouse_price_data['add_feat_5'] = house_price_data['BsmtFullBath'] + house_price_data['BsmtHalfBath']\nhouse_price_data['add_feat_6'] = house_price_data['FullBath'] + house_price_data['HalfBath']\nhouse_price_data['add_feat_7'] = house_price_data['GarageArea']/house_price_data['LotArea']\nhouse_price_data['add_feat_8'] = house_price_data['WoodDeckSF']/house_price_data['LotArea']\nhouse_price_data['add_feat_9'] = house_price_data['OpenPorchSF'] + house_price_data['EnclosedPorch'] + house_price_data['3SsnPorch'] + house_price_data['ScreenPorch']\nhouse_price_data['add_feat_10'] = house_price_data['ScreenPorch']/house_price_data['EnclosedPorch']\nhouse_price_data['add_feat_11'] = house_price_data['LotArea'] - (house_price_data['GarageArea'] + house_price_data['PoolArea'])\nhouse_price_data['add_feat_12'] = house_price_data['YrSold'] - house_price_data['YearBuilt']\nhouse_price_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:02:55.378804Z","iopub.execute_input":"2022-07-15T18:02:55.379215Z","iopub.status.idle":"2022-07-15T18:02:55.400895Z","shell.execute_reply.started":"2022-07-15T18:02:55.379183Z","shell.execute_reply":"2022-07-15T18:02:55.399972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[['add_feat_1', 'add_feat_2', 'add_feat_3', 'add_feat_4', 'add_feat_5', 'add_feat_6',\n                 'add_feat_7', 'add_feat_8', 'add_feat_9', 'add_feat_10', 'add_feat_11', 'add_feat_12']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:02.088946Z","iopub.execute_input":"2022-07-15T18:03:02.090306Z","iopub.status.idle":"2022-07-15T18:03:02.106497Z","shell.execute_reply.started":"2022-07-15T18:03:02.090249Z","shell.execute_reply":"2022-07-15T18:03:02.104852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[['add_feat_2','add_feat_10']] = house_price_data[['add_feat_2','add_feat_10']].fillna(0)\nhouse_price_data[['add_feat_2','add_feat_10']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:04.028976Z","iopub.execute_input":"2022-07-15T18:03:04.029392Z","iopub.status.idle":"2022-07-15T18:03:04.042788Z","shell.execute_reply.started":"2022-07-15T18:03:04.029359Z","shell.execute_reply":"2022-07-15T18:03:04.041325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.isinf(house_price_data[['add_feat_1', 'add_feat_2', 'add_feat_3', 'add_feat_4', 'add_feat_5', 'add_feat_6',\n                 'add_feat_7', 'add_feat_8', 'add_feat_9', 'add_feat_10', 'add_feat_11', 'add_feat_12']]).any()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:08.966393Z","iopub.execute_input":"2022-07-15T18:03:08.967533Z","iopub.status.idle":"2022-07-15T18:03:08.979374Z","shell.execute_reply.started":"2022-07-15T18:03:08.967482Z","shell.execute_reply":"2022-07-15T18:03:08.977937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data.replace([np.inf, -np.inf], 0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:12.320716Z","iopub.execute_input":"2022-07-15T18:03:12.321269Z","iopub.status.idle":"2022-07-15T18:03:12.345596Z","shell.execute_reply.started":"2022-07-15T18:03:12.321228Z","shell.execute_reply":"2022-07-15T18:03:12.344485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre processing pipeline","metadata":{}},{"cell_type":"code","source":"house_price_data[cat_feat].head().T","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:29.968806Z","iopub.execute_input":"2022-07-15T18:03:29.969168Z","iopub.status.idle":"2022-07-15T18:03:29.989289Z","shell.execute_reply.started":"2022-07-15T18:03:29.969137Z","shell.execute_reply":"2022-07-15T18:03:29.987979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = OrdinalEncoder()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:36.478452Z","iopub.execute_input":"2022-07-15T18:03:36.479081Z","iopub.status.idle":"2022-07-15T18:03:36.484232Z","shell.execute_reply.started":"2022-07-15T18:03:36.479037Z","shell.execute_reply":"2022-07-15T18:03:36.483092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder.fit(house_price_data.loc[:1460, cat_feat])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:40.968761Z","iopub.execute_input":"2022-07-15T18:03:40.969198Z","iopub.status.idle":"2022-07-15T18:03:40.987268Z","shell.execute_reply.started":"2022-07-15T18:03:40.969167Z","shell.execute_reply":"2022-07-15T18:03:40.986205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[cat_feat] = encoder.transform(house_price_data[cat_feat])\nhouse_price_data[cat_feat]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:43.242904Z","iopub.execute_input":"2022-07-15T18:03:43.243564Z","iopub.status.idle":"2022-07-15T18:03:43.331558Z","shell.execute_reply.started":"2022-07-15T18:03:43.243503Z","shell.execute_reply":"2022-07-15T18:03:43.330460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:45.497954Z","iopub.execute_input":"2022-07-15T18:03:45.498518Z","iopub.status.idle":"2022-07-15T18:03:45.503357Z","shell.execute_reply.started":"2022-07-15T18:03:45.498476Z","shell.execute_reply":"2022-07-15T18:03:45.502520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_feat","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:48.844682Z","iopub.execute_input":"2022-07-15T18:03:48.845592Z","iopub.status.idle":"2022-07-15T18:03:48.852456Z","shell.execute_reply.started":"2022-07-15T18:03:48.845531Z","shell.execute_reply":"2022-07-15T18:03:48.851502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler.fit(house_price_data.loc[:1460, num_feat])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:53.004073Z","iopub.execute_input":"2022-07-15T18:03:53.005475Z","iopub.status.idle":"2022-07-15T18:03:53.017883Z","shell.execute_reply.started":"2022-07-15T18:03:53.005411Z","shell.execute_reply":"2022-07-15T18:03:53.016897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"house_price_data[num_feat] = scaler.transform(house_price_data[num_feat])\nhouse_price_data","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:03:56.203983Z","iopub.execute_input":"2022-07-15T18:03:56.205119Z","iopub.status.idle":"2022-07-15T18:03:56.252324Z","shell.execute_reply.started":"2022-07-15T18:03:56.205078Z","shell.execute_reply":"2022-07-15T18:03:56.251554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{}},{"cell_type":"code","source":"target_max = np.max(target)\ntarget_min = np.min(target)\ntarget = (target - np.min(target))/(np.max(target) - np.min(target))\ntarget","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:14.369091Z","iopub.execute_input":"2022-07-15T18:04:14.369897Z","iopub.status.idle":"2022-07-15T18:04:14.382241Z","shell.execute_reply.started":"2022-07-15T18:04:14.369861Z","shell.execute_reply":"2022-07-15T18:04:14.380807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(house_price_data[:1460].values, target.values, test_size=0.1)\ntrain_x.shape, test_x.shape, train_y.shape, test_y.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:16.351883Z","iopub.execute_input":"2022-07-15T18:04:16.352745Z","iopub.status.idle":"2022-07-15T18:04:16.363850Z","shell.execute_reply.started":"2022-07-15T18:04:16.352697Z","shell.execute_reply":"2022-07-15T18:04:16.362334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ### cross validation","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=3)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:18.747219Z","iopub.execute_input":"2022-07-15T18:04:18.747643Z","iopub.status.idle":"2022-07-15T18:04:18.753349Z","shell.execute_reply.started":"2022-07-15T18:04:18.747597Z","shell.execute_reply":"2022-07-15T18:04:18.751809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(estimator):\n    i = 1\n    train_log = {}\n\n    for train_index, valid_index in kf.split(train_x):\n        X_train, X_val = train_x[train_index], train_x[valid_index]\n        y_train, y_val = train_y[train_index], train_y[valid_index]\n        fold_log = {}\n        \n        estimator.fit(X_train,y_train)\n\n        train_pred = estimator.predict(X_train)\n        valid_pred = estimator.predict(X_val)\n\n        fold_log['model'] = estimator\n        fold_log['train_score_r2'] = r2_score(y_true=y_train, y_pred=train_pred)\n        fold_log['train_score_ev'] = explained_variance_score(y_true=y_train, y_pred=train_pred)\n        fold_log['train_score_rmse'] = mean_squared_error(y_true=y_train, y_pred=train_pred, squared=True)\n        fold_log['valid_score_r2'] = r2_score(y_true=y_val, y_pred=valid_pred)\n        fold_log['valid_score_ev'] = explained_variance_score(y_true=y_val, y_pred=valid_pred)\n        fold_log['valid_score_rmse'] = mean_squared_error(y_true=y_val, y_pred=valid_pred, squared=True)\n\n        train_log[f'fold_{i}'] = fold_log\n        print(f'fold {i} -- done.')\n        print(f\"train r2: {fold_log['train_score_r2']} -- validation r2: {fold_log['valid_score_r2']}\")\n        print(f\"train ev: {fold_log['train_score_ev']} -- validation ev: {fold_log['valid_score_ev']}\")\n        print(f\"train rmse: {fold_log['train_score_rmse']} -- validation rmse: {fold_log['valid_score_rmse']}\")\n        print()\n\n        i += 1\n        \n    return train_log","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:22.541097Z","iopub.execute_input":"2022-07-15T18:04:22.541873Z","iopub.status.idle":"2022-07-15T18:04:22.553444Z","shell.execute_reply.started":"2022-07-15T18:04:22.541837Z","shell.execute_reply":"2022-07-15T18:04:22.552258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear Estimators","metadata":{}},{"cell_type":"code","source":"linear_est = LinearRegression(n_jobs=-1)\nl1_est = Lasso()\nl2_est = Ridge()\nelnet_est = ElasticNet()\nransac_est = RANSACRegressor()\nhuber_est = HuberRegressor()\nsgd_est = SGDRegressor(early_stopping=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:25.979521Z","iopub.execute_input":"2022-07-15T18:04:25.980207Z","iopub.status.idle":"2022-07-15T18:04:25.985258Z","shell.execute_reply.started":"2022-07-15T18:04:25.980174Z","shell.execute_reply":"2022-07-15T18:04:25.984351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyper-parameter distributions","metadata":{}},{"cell_type":"code","source":"alpha_dis = {'alpha':np.random.uniform(0,1,12)}\nelnet_dis = {'alpha':np.random.uniform(0,1,12),\n             'l1_ratio':np.random.uniform(0.1,0.9,12)}\nsgd_dis = {'penalty':['l2', 'l1', 'elasticnet'],\n           'l1_ratio':np.random.uniform(0.1,0.9,12),\n           'learning_rate':['constant','optimal','invscaling','adaptive']}","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:27.670207Z","iopub.execute_input":"2022-07-15T18:04:27.670993Z","iopub.status.idle":"2022-07-15T18:04:27.676670Z","shell.execute_reply.started":"2022-07-15T18:04:27.670955Z","shell.execute_reply":"2022-07-15T18:04:27.675753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Grid Search","metadata":{}},{"cell_type":"code","source":"cv = 5\n\nl1_grid_search = RandomizedSearchCV(estimator=l1_est,\n                                    param_distributions=alpha_dis,\n                                    scoring={'r2': make_scorer(r2_score),\n                                             'mse': make_scorer(mean_squared_error)\n                                            },\n                                    refit='mse',\n                                    n_jobs=-1,\n                                    cv=cv)\n\nl2_grid_search = RandomizedSearchCV(estimator=l2_est,\n                                    param_distributions=alpha_dis,\n                                    scoring={'r2': make_scorer(r2_score),\n                                             'mse': make_scorer(mean_squared_error)\n                                            },\n                                    refit='mse',\n                                    n_jobs=-1,\n                                    cv=cv)\n\nelnet_est_grid_search = RandomizedSearchCV(estimator=elnet_est,\n                                           param_distributions=elnet_dis,\n                                           scoring={'r2': make_scorer(r2_score),\n                                                    'mse': make_scorer(mean_squared_error)\n                                                   },\n                                           refit='mse',\n                                           n_jobs=-1,\n                                           cv=cv)\n\nsgd_est_grid_search = RandomizedSearchCV(estimator=sgd_est,\n                                         param_distributions=sgd_dis,\n                                         scoring={'r2': make_scorer(r2_score),\n                                                  'mse': make_scorer(mean_squared_error)\n                                                 },\n                                         refit='mse',\n                                         n_jobs=-1,\n                                         cv=cv)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:31.947007Z","iopub.execute_input":"2022-07-15T18:04:31.947727Z","iopub.status.idle":"2022-07-15T18:04:31.958765Z","shell.execute_reply.started":"2022-07-15T18:04:31.947691Z","shell.execute_reply":"2022-07-15T18:04:31.957761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_grid_search.fit(train_x,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:41.940715Z","iopub.execute_input":"2022-07-15T18:04:41.941920Z","iopub.status.idle":"2022-07-15T18:04:43.334635Z","shell.execute_reply.started":"2022-07-15T18:04:41.941866Z","shell.execute_reply":"2022-07-15T18:04:43.333135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l2_grid_search.fit(train_x,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:45.059683Z","iopub.execute_input":"2022-07-15T18:04:45.060373Z","iopub.status.idle":"2022-07-15T18:04:45.250442Z","shell.execute_reply.started":"2022-07-15T18:04:45.060314Z","shell.execute_reply":"2022-07-15T18:04:45.249197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"elnet_est_grid_search.fit(train_x,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:46.970485Z","iopub.execute_input":"2022-07-15T18:04:46.970874Z","iopub.status.idle":"2022-07-15T18:04:47.163797Z","shell.execute_reply.started":"2022-07-15T18:04:46.970844Z","shell.execute_reply":"2022-07-15T18:04:47.162581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgd_est_grid_search.fit(train_x,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:48.470361Z","iopub.execute_input":"2022-07-15T18:04:48.471090Z","iopub.status.idle":"2022-07-15T18:04:48.970743Z","shell.execute_reply.started":"2022-07-15T18:04:48.471039Z","shell.execute_reply":"2022-07-15T18:04:48.969688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_grid_search.best_params_, l1_grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:53.089655Z","iopub.execute_input":"2022-07-15T18:04:53.090036Z","iopub.status.idle":"2022-07-15T18:04:53.096192Z","shell.execute_reply.started":"2022-07-15T18:04:53.090004Z","shell.execute_reply":"2022-07-15T18:04:53.095384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l2_grid_search.best_params_, l2_grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:54.556641Z","iopub.execute_input":"2022-07-15T18:04:54.557230Z","iopub.status.idle":"2022-07-15T18:04:54.562985Z","shell.execute_reply.started":"2022-07-15T18:04:54.557198Z","shell.execute_reply":"2022-07-15T18:04:54.561975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"elnet_est_grid_search.best_params_, elnet_est_grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:56.216108Z","iopub.execute_input":"2022-07-15T18:04:56.216953Z","iopub.status.idle":"2022-07-15T18:04:56.223125Z","shell.execute_reply.started":"2022-07-15T18:04:56.216920Z","shell.execute_reply":"2022-07-15T18:04:56.222056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgd_est_grid_search.best_params_, sgd_est_grid_search.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:04:57.401653Z","iopub.execute_input":"2022-07-15T18:04:57.401999Z","iopub.status.idle":"2022-07-15T18:04:57.409299Z","shell.execute_reply.started":"2022-07-15T18:04:57.401972Z","shell.execute_reply":"2022-07-15T18:04:57.408037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train estimators","metadata":{}},{"cell_type":"code","source":"linear_log = train(linear_est)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:06.658777Z","iopub.execute_input":"2022-07-15T18:05:06.659634Z","iopub.status.idle":"2022-07-15T18:05:06.713817Z","shell.execute_reply.started":"2022-07-15T18:05:06.659584Z","shell.execute_reply":"2022-07-15T18:05:06.712559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_log = train(Lasso(0.6304431980330377))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:10.597830Z","iopub.execute_input":"2022-07-15T18:05:10.598222Z","iopub.status.idle":"2022-07-15T18:05:10.631359Z","shell.execute_reply.started":"2022-07-15T18:05:10.598193Z","shell.execute_reply":"2022-07-15T18:05:10.630080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l2_log = train(Ridge(0.6304431980330377))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:14.641002Z","iopub.execute_input":"2022-07-15T18:05:14.641463Z","iopub.status.idle":"2022-07-15T18:05:14.707097Z","shell.execute_reply.started":"2022-07-15T18:05:14.641425Z","shell.execute_reply":"2022-07-15T18:05:14.705221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"elnet_log = train(ElasticNet(l1_ratio=0.5238076223695187, alpha=0.6266053330663998))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:19.377485Z","iopub.execute_input":"2022-07-15T18:05:19.377936Z","iopub.status.idle":"2022-07-15T18:05:19.440438Z","shell.execute_reply.started":"2022-07-15T18:05:19.377895Z","shell.execute_reply":"2022-07-15T18:05:19.437719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ransac_log = train(RANSACRegressor())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:23.479720Z","iopub.execute_input":"2022-07-15T18:05:23.480156Z","iopub.status.idle":"2022-07-15T18:05:25.900306Z","shell.execute_reply.started":"2022-07-15T18:05:23.480124Z","shell.execute_reply":"2022-07-15T18:05:25.898952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"huber_log = train(HuberRegressor())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:31.428149Z","iopub.execute_input":"2022-07-15T18:05:31.428535Z","iopub.status.idle":"2022-07-15T18:05:31.764091Z","shell.execute_reply.started":"2022-07-15T18:05:31.428504Z","shell.execute_reply":"2022-07-15T18:05:31.762845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgd_log = train(SGDRegressor(penalty='l1',\n                             learning_rate='optimal',\n                             l1_ratio=0.1169569908594938,\n                             early_stopping=True))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:36.588495Z","iopub.execute_input":"2022-07-15T18:05:36.589502Z","iopub.status.idle":"2022-07-15T18:05:36.674885Z","shell.execute_reply.started":"2022-07-15T18:05:36.589453Z","shell.execute_reply":"2022-07-15T18:05:36.673620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_linear = np.zeros(test_y.shape[0])\n\nfor fold in linear_log.keys():\n    test_pred_linear += linear_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:42.188869Z","iopub.execute_input":"2022-07-15T18:05:42.189280Z","iopub.status.idle":"2022-07-15T18:05:42.206165Z","shell.execute_reply.started":"2022-07-15T18:05:42.189247Z","shell.execute_reply":"2022-07-15T18:05:42.204346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_l1 = np.zeros(test_y.shape[0])\n\nfor fold in l1_log.keys():\n    test_pred_l1 += l1_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:45.395667Z","iopub.execute_input":"2022-07-15T18:05:45.396059Z","iopub.status.idle":"2022-07-15T18:05:45.407469Z","shell.execute_reply.started":"2022-07-15T18:05:45.396029Z","shell.execute_reply":"2022-07-15T18:05:45.405845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_l2 = np.zeros(test_y.shape[0])\n\nfor fold in l2_log.keys():\n    test_pred_l2 += l2_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:46.600047Z","iopub.execute_input":"2022-07-15T18:05:46.600591Z","iopub.status.idle":"2022-07-15T18:05:46.614155Z","shell.execute_reply.started":"2022-07-15T18:05:46.600521Z","shell.execute_reply":"2022-07-15T18:05:46.612403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_elnet = np.zeros(test_y.shape[0])\n\nfor fold in elnet_log.keys():\n    test_pred_elnet += elnet_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:48.510096Z","iopub.execute_input":"2022-07-15T18:05:48.510532Z","iopub.status.idle":"2022-07-15T18:05:48.520219Z","shell.execute_reply.started":"2022-07-15T18:05:48.510495Z","shell.execute_reply":"2022-07-15T18:05:48.518604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_ransac = np.zeros(test_y.shape[0])\n\nfor fold in ransac_log.keys():\n    test_pred_ransac += ransac_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:50.442780Z","iopub.execute_input":"2022-07-15T18:05:50.443237Z","iopub.status.idle":"2022-07-15T18:05:50.451518Z","shell.execute_reply.started":"2022-07-15T18:05:50.443199Z","shell.execute_reply":"2022-07-15T18:05:50.450243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_huber = np.zeros(test_y.shape[0])\n\nfor fold in huber_log.keys():\n    test_pred_huber += huber_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:51.897170Z","iopub.execute_input":"2022-07-15T18:05:51.897643Z","iopub.status.idle":"2022-07-15T18:05:51.908314Z","shell.execute_reply.started":"2022-07-15T18:05:51.897604Z","shell.execute_reply":"2022-07-15T18:05:51.906901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_sgd = np.zeros(test_y.shape[0])\n\nfor fold in sgd_log.keys():\n    test_pred_sgd += sgd_log[fold]['model'].predict(test_x)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:53.627063Z","iopub.execute_input":"2022-07-15T18:05:53.627526Z","iopub.status.idle":"2022-07-15T18:05:53.639737Z","shell.execute_reply.started":"2022-07-15T18:05:53.627481Z","shell.execute_reply":"2022-07-15T18:05:53.637532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_linear), mean_squared_error(test_y, test_pred_linear)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:55.938782Z","iopub.execute_input":"2022-07-15T18:05:55.940054Z","iopub.status.idle":"2022-07-15T18:05:55.948140Z","shell.execute_reply.started":"2022-07-15T18:05:55.939978Z","shell.execute_reply":"2022-07-15T18:05:55.946788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_l1), mean_squared_error(test_y, test_pred_l1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:05:59.335974Z","iopub.execute_input":"2022-07-15T18:05:59.336348Z","iopub.status.idle":"2022-07-15T18:05:59.344591Z","shell.execute_reply.started":"2022-07-15T18:05:59.336318Z","shell.execute_reply":"2022-07-15T18:05:59.343584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_l2), mean_squared_error(test_y, test_pred_l2)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:00.530031Z","iopub.execute_input":"2022-07-15T18:06:00.531488Z","iopub.status.idle":"2022-07-15T18:06:00.539555Z","shell.execute_reply.started":"2022-07-15T18:06:00.531444Z","shell.execute_reply":"2022-07-15T18:06:00.538658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_elnet), mean_squared_error(test_y, test_pred_elnet)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:01.630263Z","iopub.execute_input":"2022-07-15T18:06:01.630968Z","iopub.status.idle":"2022-07-15T18:06:01.639777Z","shell.execute_reply.started":"2022-07-15T18:06:01.630922Z","shell.execute_reply":"2022-07-15T18:06:01.638271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_ransac), mean_squared_error(test_y, test_pred_ransac)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:04.690026Z","iopub.execute_input":"2022-07-15T18:06:04.690478Z","iopub.status.idle":"2022-07-15T18:06:04.699563Z","shell.execute_reply.started":"2022-07-15T18:06:04.690441Z","shell.execute_reply":"2022-07-15T18:06:04.698635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_huber), mean_squared_error(test_y, test_pred_huber)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:05.868101Z","iopub.execute_input":"2022-07-15T18:06:05.869419Z","iopub.status.idle":"2022-07-15T18:06:05.876764Z","shell.execute_reply.started":"2022-07-15T18:06:05.869381Z","shell.execute_reply":"2022-07-15T18:06:05.875468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2_score(test_y, test_pred_sgd), mean_squared_error(test_y, test_pred_sgd)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:06.865831Z","iopub.execute_input":"2022-07-15T18:06:06.866742Z","iopub.status.idle":"2022-07-15T18:06:06.874053Z","shell.execute_reply.started":"2022-07-15T18:06:06.866703Z","shell.execute_reply":"2022-07-15T18:06:06.873194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6), dpi=300)\nsns.scatterplot(x=test_y, y=test_pred_linear, label='linear')\nsns.scatterplot(x=test_y, y=test_pred_l1, label='l1')\nsns.scatterplot(x=test_y, y=test_pred_l2, label='l2')\nsns.scatterplot(x=test_y, y=test_pred_elnet, label='elastic net')\nsns.scatterplot(x=test_y, y=test_pred_ransac, label='ransac')\nsns.scatterplot(x=test_y, y=test_pred_huber, label='huber')\nplt.xlabel('ground truth')\nplt.ylabel('predicted')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:14.807236Z","iopub.execute_input":"2022-07-15T18:06:14.808200Z","iopub.status.idle":"2022-07-15T18:06:15.523511Z","shell.execute_reply.started":"2022-07-15T18:06:14.808160Z","shell.execute_reply":"2022-07-15T18:06:15.522300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Selection","metadata":{}},{"cell_type":"code","source":"models = [('OLS', linear_est),\n          ('Lasso', Lasso(0.6304431980330377)),\n          ('Ridge', Ridge(0.6304431980330377)),\n          ('ElasticNet', ElasticNet(l1_ratio=0.5238076223695187, alpha=0.6266053330663998)),\n          ('RANSAC', RANSACRegressor()),\n          ('Huber', HuberRegressor())]\nresults = []\nnames = []\n\nfor name, model in models:\n    kfold = KFold(n_splits=3)\n    cv_results = cross_val_score(model, test_x, test_y, cv=kfold, scoring='r2')\n    results.append(cv_results)\n    names.append(name)\n    msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n    print(msg)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:39.808755Z","iopub.execute_input":"2022-07-15T18:06:39.809682Z","iopub.status.idle":"2022-07-15T18:06:40.212163Z","shell.execute_reply.started":"2022-07-15T18:06:39.809639Z","shell.execute_reply":"2022-07-15T18:06:40.210871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure()\nfig.suptitle('Algorithm Comparison')\nax = fig.add_subplot(111)\nplt.boxplot(results)\nax.set_xticklabels(names)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:06:50.509445Z","iopub.execute_input":"2022-07-15T18:06:50.510607Z","iopub.status.idle":"2022-07-15T18:06:50.666932Z","shell.execute_reply.started":"2022-07-15T18:06:50.510561Z","shell.execute_reply":"2022-07-15T18:06:50.665924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"test_pred_ridge = np.zeros(test_data.shape[0])\n\nfor fold in l2_log.keys():\n    test_pred_ridge += l2_log[fold]['model'].predict(house_price_data[1460:].values)/3","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:09:26.083003Z","iopub.execute_input":"2022-07-15T18:09:26.085917Z","iopub.status.idle":"2022-07-15T18:09:26.103837Z","shell.execute_reply.started":"2022-07-15T18:09:26.085869Z","shell.execute_reply":"2022-07-15T18:09:26.102519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/house-prices-advanced-regression-techniques/sample_submission.csv', index_col='Id')\nsubmission['SalePrice'] = test_pred_ridge * (target_max - target_min) + target_min\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:10:46.089606Z","iopub.execute_input":"2022-07-15T18:10:46.090986Z","iopub.status.idle":"2022-07-15T18:10:46.108906Z","shell.execute_reply.started":"2022-07-15T18:10:46.090920Z","shell.execute_reply":"2022-07-15T18:10:46.107809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## make submission file","metadata":{}},{"cell_type":"code","source":"submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:10:50.287513Z","iopub.execute_input":"2022-07-15T18:10:50.288672Z","iopub.status.idle":"2022-07-15T18:10:50.302271Z","shell.execute_reply.started":"2022-07-15T18:10:50.288624Z","shell.execute_reply":"2022-07-15T18:10:50.300807Z"},"trusted":true},"execution_count":null,"outputs":[]}]}