{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style=\"border-radius:10px;\n            border : #015a2c solid;\n            background-color:#ecfff5;\n           font-size:110%;\n           letter-spacing:0.5px;\n            text-align: center\">\n\n<center><h1 style=\"padding: 25px 0px; color:#015a2c; font-weight: bold; font-family: Cursive\">\nHouse price prediction ⛪🏤🏡</h1></center>\n<center><h3 style=\"padding-bottom: 25px; color:#015a2c; font-weight: bold; font-style:italic; font-family: Cursive\">\n(linear and nonlinear models)</h3></center>     \n\n</div>","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T09:43:05.728606Z","iopub.execute_input":"2022-07-15T09:43:05.728869Z","iopub.status.idle":"2022-07-15T09:43:05.736293Z","shell.execute_reply.started":"2022-07-15T09:43:05.728841Z","shell.execute_reply":"2022-07-15T09:43:05.735383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport seaborn as sns\n\n# ----------------------------------------------------\nimport sklearn\nimport scipy\nimport statsmodels.api as sm \nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n\n# ----------------------------------------------------\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.feature_selection import SelectKBest, SelectPercentile\n\n# ----------------------------------------------------\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV, RandomizedSearchCV\nfrom sklearn.linear_model import LinearRegression, ElasticNet, Ridge, Lasso\n\n# ----------------------------------------------------\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\nfrom sklearn.ensemble import ExtraTreesRegressor, BaggingRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n\n# ----------------------------------------------------\nsns.set()\nsns.set_style(\"white\")\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:05.764751Z","iopub.execute_input":"2022-07-15T09:43:05.765075Z","iopub.status.idle":"2022-07-15T09:43:08.201029Z","shell.execute_reply.started":"2022-07-15T09:43:05.765039Z","shell.execute_reply":"2022-07-15T09:43:08.200025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Functions","metadata":{}},{"cell_type":"code","source":"def outlier_detect_new(df, col):\n    q1_col = Q1[col]\n    iqr_col = IQR[col]\n    q3_col = Q3[col]\n    return df[((df[col] < (q1_col - 1.5 * iqr_col)) |(df[col] > (q3_col + 1.5 * iqr_col)))]\n\n# ----------------------------------------------------------\ndef lower_outlier(df, col):\n    q1_col = Q1[col]\n    iqr_col = IQR[col]\n    q3_col = Q3[col]\n    lower = df[(df[col] < (q1_col - 1.5 * iqr_col))]\n    return lower\n\n# ----------------------------------------------------------\ndef upper_outlier(df, col):\n    q1_col = Q1[col]\n    iqr_col = IQR[col]\n    q3_col = Q3[col]\n    upper = df[(df[col] > (q3_col + 1.5 * iqr_col))]\n    return upper\n\n# ----------------------------------------------------------\ndef preprocess(df, col):\n    print(\"******************** {} ********************\\n\".format(col))\n    plt.figure(figsize=(16,4))\n    plt.subplot(1,2,1)\n    df[col].plot(kind='box', subplots=True, sharex=False, vert=False)\n    plt.subplot(1,2,2)\n    df[col].plot(kind='density', subplots=True, sharex=False)\n    plt.show()\n\n# ----------------------------------------------------------\ndef preprocess_cat(df, col):\n    print(\"******************** {} ********************\\n\".format(col))\n    df[col].value_counts().plot(kind='bar')\n    plt.xticks(rotation='vertical')\n    plt.show()\n    \n# ----------------------------------------------------------\ndef replace_upper(df, col):\n    q1_col = Q1[col]\n    iqr_col = IQR[col]\n    q3_col = Q3[col]\n    tmp = 9999999\n    upper = q3_col + 1.5 * iqr_col\n    df[col] = df[col].where(lambda x: (x < (upper)), tmp)\n    df[col] = df[col].replace(tmp, upper)\n\n# ----------------------------------------------------------\ndef replace_lower(df, col):\n    q1_col = Q1[col]\n    iqr_col = IQR[col]\n    q3_col = Q3[col]\n    tmp = 1111111\n    lower = q1_col - 1.5 * iqr_col\n    df[col] = df[col].where(lambda x: (x > (lower)), tmp)\n    df[col] = df[col].replace(tmp, lower)\n\n# ----------------------------------------------------------\ndef replace_mode(df, col):\n    df[col] = df[col].fillna(df[col].mode()[0])\n\n# ----------------------------------------------------------\ndef build_model(x,y):\n    x = sm.add_constant(x)\n    lm = sm.OLS(y,x).fit()\n    print(lm.summary())\n    return x, lm\n\n# ----------------------------------------------------------\ndef checkVIF(X):\n    vif = pd.DataFrame()\n    vif['Features'] = X.columns\n    vif['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]\n    vif['VIF'] = round(vif['VIF'], 2)\n    vif = vif.sort_values(by = \"VIF\", ascending = False)\n    return(vif)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.202991Z","iopub.execute_input":"2022-07-15T09:43:08.203240Z","iopub.status.idle":"2022-07-15T09:43:08.228684Z","shell.execute_reply.started":"2022-07-15T09:43:08.203209Z","shell.execute_reply":"2022-07-15T09:43:08.227730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Understanding","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/home-data-for-ml-course/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/home-data-for-ml-course/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.230030Z","iopub.execute_input":"2022-07-15T09:43:08.230521Z","iopub.status.idle":"2022-07-15T09:43:08.334817Z","shell.execute_reply.started":"2022-07-15T09:43:08.230470Z","shell.execute_reply":"2022-07-15T09:43:08.333755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.338534Z","iopub.execute_input":"2022-07-15T09:43:08.338849Z","iopub.status.idle":"2022-07-15T09:43:08.379333Z","shell.execute_reply.started":"2022-07-15T09:43:08.338816Z","shell.execute_reply":"2022-07-15T09:43:08.378540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.380446Z","iopub.execute_input":"2022-07-15T09:43:08.380695Z","iopub.status.idle":"2022-07-15T09:43:08.407643Z","shell.execute_reply.started":"2022-07-15T09:43:08.380665Z","shell.execute_reply":"2022-07-15T09:43:08.406575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train: {}\".format(train.shape))\nprint(\"test: {}\".format(test.shape))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.409268Z","iopub.execute_input":"2022-07-15T09:43:08.409748Z","iopub.status.idle":"2022-07-15T09:43:08.421980Z","shell.execute_reply.started":"2022-07-15T09:43:08.409710Z","shell.execute_reply":"2022-07-15T09:43:08.421147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.422919Z","iopub.execute_input":"2022-07-15T09:43:08.423137Z","iopub.status.idle":"2022-07-15T09:43:08.550845Z","shell.execute_reply.started":"2022-07-15T09:43:08.423101Z","shell.execute_reply":"2022-07-15T09:43:08.549864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train: {}\".format(train.info()))\nprint(\"\\n************************************************************\\n\")\nprint(\"test: {}\".format(test.info()))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.552111Z","iopub.execute_input":"2022-07-15T09:43:08.552328Z","iopub.status.idle":"2022-07-15T09:43:08.600766Z","shell.execute_reply.started":"2022-07-15T09:43:08.552301Z","shell.execute_reply":"2022-07-15T09:43:08.599947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', train.shape[0])\npd.set_option('display.max_rows', test.shape[0])\npd.DataFrame(train.isnull().sum().sort_values(ascending = False))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.601957Z","iopub.execute_input":"2022-07-15T09:43:08.602239Z","iopub.status.idle":"2022-07-15T09:43:08.627224Z","shell.execute_reply.started":"2022-07-15T09:43:08.602207Z","shell.execute_reply":"2022-07-15T09:43:08.626523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(test.isnull().sum().sort_values(ascending = False))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.630981Z","iopub.execute_input":"2022-07-15T09:43:08.631402Z","iopub.status.idle":"2022-07-15T09:43:08.655204Z","shell.execute_reply.started":"2022-07-15T09:43:08.631350Z","shell.execute_reply":"2022-07-15T09:43:08.654567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.plot(kind='density', subplots=True, layout=(14,6), sharex=False, figsize= (40,80))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:08.656277Z","iopub.execute_input":"2022-07-15T09:43:08.656530Z","iopub.status.idle":"2022-07-15T09:43:18.780861Z","shell.execute_reply.started":"2022-07-15T09:43:08.656502Z","shell.execute_reply":"2022-07-15T09:43:18.779709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"train.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', \n            'FireplaceQu', 'LotFrontage'], axis=1, inplace=True)\n\ntest.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', \n           'FireplaceQu', 'LotFrontage'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.782336Z","iopub.execute_input":"2022-07-15T09:43:18.783076Z","iopub.status.idle":"2022-07-15T09:43:18.793843Z","shell.execute_reply.started":"2022-07-15T09:43:18.783024Z","shell.execute_reply":"2022-07-15T09:43:18.793172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Q1 = train.quantile(0.25)\nQ3 = train.quantile(0.75)\nIQR = Q3 - Q1","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.795197Z","iopub.execute_input":"2022-07-15T09:43:18.795645Z","iopub.status.idle":"2022-07-15T09:43:18.809645Z","shell.execute_reply.started":"2022-07-15T09:43:18.795613Z","shell.execute_reply":"2022-07-15T09:43:18.808724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_feature = train.dtypes==object\nfinal_categorical_feature = train.columns[categorical_feature].tolist()\n\n#----------------------------------------------------\ncategorical_feature_test = test.dtypes==object\nfinal_categorical_feature_test = test.columns[categorical_feature_test].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.811299Z","iopub.execute_input":"2022-07-15T09:43:18.811781Z","iopub.status.idle":"2022-07-15T09:43:18.818528Z","shell.execute_reply.started":"2022-07-15T09:43:18.811747Z","shell.execute_reply":"2022-07-15T09:43:18.817681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_feature = train.dtypes!=object\nfinal_numeric_feature = train.columns[numeric_feature].tolist()\n\n#----------------------------------------------------\nnumeric_feature_test = test.dtypes!=object\nfinal_numeric_feature_test = test.columns[numeric_feature_test].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.820940Z","iopub.execute_input":"2022-07-15T09:43:18.821288Z","iopub.status.idle":"2022-07-15T09:43:18.830689Z","shell.execute_reply.started":"2022-07-15T09:43:18.821242Z","shell.execute_reply":"2022-07-15T09:43:18.829899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"border-radius:10px;\n            background-color:#ffffff;\n            letter-spacing:0.5px;\">\n\n<h3 style=\"padding: 5px 0px; color:#015a2c; font-weight: bold; font-family: Cursive\">\n1. Numerical Field</h3>\n</div>","metadata":{}},{"cell_type":"code","source":"numeric = train[final_numeric_feature]\n\n#-------------------------------------------\nnumeric_test = test[final_numeric_feature_test]\n\nnumeric.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.832120Z","iopub.execute_input":"2022-07-15T09:43:18.832638Z","iopub.status.idle":"2022-07-15T09:43:18.867874Z","shell.execute_reply.started":"2022-07-15T09:43:18.832592Z","shell.execute_reply":"2022-07-15T09:43:18.867074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_names = numeric.columns\ncol_names_test = numeric_test.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.869104Z","iopub.execute_input":"2022-07-15T09:43:18.869351Z","iopub.status.idle":"2022-07-15T09:43:18.879351Z","shell.execute_reply.started":"2022-07-15T09:43:18.869318Z","shell.execute_reply":"2022-07-15T09:43:18.878664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(col_names)):\n    preprocess(train, col_names[i])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:18.880702Z","iopub.execute_input":"2022-07-15T09:43:18.880956Z","iopub.status.idle":"2022-07-15T09:43:34.920489Z","shell.execute_reply.started":"2022-07-15T09:43:18.880925Z","shell.execute_reply":"2022-07-15T09:43:34.919352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train\")\nfor i in range(len(col_names)):\n    print(\"{}: {}\".format(col_names[i],(outlier_detect_new(numeric,col_names[i]).shape[0])))\n    \nprint(\"***********************************\")\nprint(\"test \")\nfor i in range(len(numeric_test.columns)):\n    print(\"{}: {}\".format(col_names_test[i],(outlier_detect_new(numeric_test,col_names_test[i]).shape[0])))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:34.921936Z","iopub.execute_input":"2022-07-15T09:43:34.922322Z","iopub.status.idle":"2022-07-15T09:43:35.004543Z","shell.execute_reply.started":"2022-07-15T09:43:34.922286Z","shell.execute_reply":"2022-07-15T09:43:35.003613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discrete_features = ['BsmtFinSF2' ,'LowQualFinSF' ,'BsmtHalfBath' ,'BedroomAbvGr' ,'KitchenAbvGr' ,\n                   'EnclosedPorch' ,'3SsnPorch' ,'ScreenPorch' ,'PoolArea' ,'MiscVal' ]\n\ndiscrete = numeric[discrete_features]\n#------------------------------------------------------------------------------------------------------------------\ndiscrete_test = numeric_test[discrete_features]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.005743Z","iopub.execute_input":"2022-07-15T09:43:35.005969Z","iopub.status.idle":"2022-07-15T09:43:35.014046Z","shell.execute_reply.started":"2022-07-15T09:43:35.005943Z","shell.execute_reply":"2022-07-15T09:43:35.013071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"continuous = numeric[['MSSubClass', 'LotArea', 'OverallQual', 'OverallCond',\n                   'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',\n                   'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',\n                   'GrLivArea', 'BsmtFullBath', 'FullBath', 'HalfBath',\n                   'TotRmsAbvGrd', 'Fireplaces','GarageYrBlt', 'GarageCars', \n                   'GarageArea', 'WoodDeckSF', 'OpenPorchSF',\n                   'MoSold', 'YrSold', 'SalePrice']]\n\ncontinuous_test = numeric_test[['MSSubClass', 'LotArea', 'OverallQual', 'OverallCond',\n                   'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',\n                   'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',\n                   'GrLivArea', 'BsmtFullBath', 'FullBath', 'HalfBath',\n                   'TotRmsAbvGrd', 'Fireplaces','GarageYrBlt', 'GarageCars', \n                   'GarageArea', 'WoodDeckSF', 'OpenPorchSF',\n                   'MoSold', 'YrSold']]\ncontinuous.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.015386Z","iopub.execute_input":"2022-07-15T09:43:35.015768Z","iopub.status.idle":"2022-07-15T09:43:35.033768Z","shell.execute_reply.started":"2022-07-15T09:43:35.015722Z","shell.execute_reply":"2022-07-15T09:43:35.033073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"continuous_col = continuous.columns\nfor i in range(len(continuous_col)):\n    replace_upper(numeric, continuous_col[i])   \n    \n#------------------------------------------------------\ncontinuous_col_test = continuous_test.columns\nfor i in range(len(continuous_col_test)):\n    replace_upper(numeric_test, continuous_col_test[i])   ","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.035477Z","iopub.execute_input":"2022-07-15T09:43:35.035736Z","iopub.status.idle":"2022-07-15T09:43:35.107580Z","shell.execute_reply.started":"2022-07-15T09:43:35.035707Z","shell.execute_reply":"2022-07-15T09:43:35.106570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train\")\nfor i in range(len(continuous_col)):\n    print(\"{}: {}\".format(continuous_col[i],(upper_outlier(numeric,continuous_col[i]).shape[0])))\n    \nprint(\"***************************************\")\n\nprint(\"test\")\nfor i in range(len(continuous_col_test)):\n    print(\"{}: {}\".format(continuous_col_test[i],(upper_outlier(numeric_test,continuous_col_test[i]).shape[0])))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.109003Z","iopub.execute_input":"2022-07-15T09:43:35.109226Z","iopub.status.idle":"2022-07-15T09:43:35.158191Z","shell.execute_reply.started":"2022-07-15T09:43:35.109199Z","shell.execute_reply":"2022-07-15T09:43:35.157251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(continuous_col)):\n    replace_lower(numeric, continuous_col[i])\n    \n#--------------------------------------------------\nfor i in range(len(continuous_col_test)):\n    replace_lower(numeric_test, continuous_col_test[i])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.159690Z","iopub.execute_input":"2022-07-15T09:43:35.160443Z","iopub.status.idle":"2022-07-15T09:43:35.213998Z","shell.execute_reply.started":"2022-07-15T09:43:35.160393Z","shell.execute_reply":"2022-07-15T09:43:35.213188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train\")\nfor i in range(len(continuous_col)):\n    print(\"{}: {}\".format(continuous_col[i],(lower_outlier(numeric,continuous_col[i]).shape[0])))\n    \nprint(\"***********************************\")\n\nprint(\"test\")\nfor i in range(len(continuous_col_test)):\n    print(\"{}: {}\".format(continuous_col_test[i],(lower_outlier(numeric_test,continuous_col_test[i]).shape[0])))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.215132Z","iopub.execute_input":"2022-07-15T09:43:35.215501Z","iopub.status.idle":"2022-07-15T09:43:35.259384Z","shell.execute_reply.started":"2022-07-15T09:43:35.215470Z","shell.execute_reply":"2022-07-15T09:43:35.258481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train:\\n\",numeric.isnull().sum())\nprint(\"*********************************\")\nprint(\"test:\\n\",numeric_test.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.260816Z","iopub.execute_input":"2022-07-15T09:43:35.261040Z","iopub.status.idle":"2022-07-15T09:43:35.272186Z","shell.execute_reply.started":"2022-07-15T09:43:35.261011Z","shell.execute_reply":"2022-07-15T09:43:35.271152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"border-radius:10px;\n            background-color:#ffffff;\n            letter-spacing:0.5px;\">\n\n<h3 style=\"padding: 5px 0px; color:#015a2c; font-weight: bold; font-family: Cursive\">\n2. Categorical feilds</h3>\n</div>","metadata":{}},{"cell_type":"code","source":"categorical = train[final_categorical_feature]\n\n#---------------------------------------------\ncategorical_test = test[final_categorical_feature_test]\ncategorical.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.273631Z","iopub.execute_input":"2022-07-15T09:43:35.273872Z","iopub.status.idle":"2022-07-15T09:43:35.304163Z","shell.execute_reply.started":"2022-07-15T09:43:35.273841Z","shell.execute_reply":"2022-07-15T09:43:35.303204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train: \\n\",categorical.isnull().sum())\nprint(\"***********************************\")\nprint(\"test: \\n\",categorical_test.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.310224Z","iopub.execute_input":"2022-07-15T09:43:35.310544Z","iopub.status.idle":"2022-07-15T09:43:35.337393Z","shell.execute_reply.started":"2022-07-15T09:43:35.310509Z","shell.execute_reply":"2022-07-15T09:43:35.336501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_names_cat = categorical.columns\n#-----------------------------------------------------------\ncol_names_cat_test = categorical_test.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.339068Z","iopub.execute_input":"2022-07-15T09:43:35.339977Z","iopub.status.idle":"2022-07-15T09:43:35.345283Z","shell.execute_reply.started":"2022-07-15T09:43:35.339918Z","shell.execute_reply":"2022-07-15T09:43:35.344092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(col_names_cat)):\n    preprocess_cat(categorical, col_names_cat[i])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:35.347152Z","iopub.execute_input":"2022-07-15T09:43:35.347462Z","iopub.status.idle":"2022-07-15T09:43:43.377207Z","shell.execute_reply.started":"2022-07-15T09:43:35.347418Z","shell.execute_reply":"2022-07-15T09:43:43.376147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(col_names_cat)):\n    replace_mode(categorical, col_names_cat[i])\n    \nfor i in range(len(col_names_cat_test)):\n    replace_mode(categorical_test, col_names_cat_test[i])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.378553Z","iopub.execute_input":"2022-07-15T09:43:43.378823Z","iopub.status.idle":"2022-07-15T09:43:43.463336Z","shell.execute_reply.started":"2022-07-15T09:43:43.378791Z","shell.execute_reply":"2022-07-15T09:43:43.462650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train: \\n\",categorical.isnull().sum())\nprint(\"***********************************\")\nprint(\"test: \\n\",categorical_test.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.464475Z","iopub.execute_input":"2022-07-15T09:43:43.464834Z","iopub.status.idle":"2022-07-15T09:43:43.487817Z","shell.execute_reply.started":"2022-07-15T09:43:43.464804Z","shell.execute_reply":"2022-07-15T09:43:43.487069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder() \ncategorical = categorical.apply(lambda col: le.fit_transform(col)) \ncategorical_test = categorical_test.apply(lambda col: le.fit_transform(col)) \ncategorical.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.488781Z","iopub.execute_input":"2022-07-15T09:43:43.489429Z","iopub.status.idle":"2022-07-15T09:43:43.554392Z","shell.execute_reply.started":"2022-07-15T09:43:43.489388Z","shell.execute_reply":"2022-07-15T09:43:43.553682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[col_names_cat] = categorical[col_names_cat]\ntest[col_names_cat_test] = categorical_test[col_names_cat_test]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.555388Z","iopub.execute_input":"2022-07-15T09:43:43.556080Z","iopub.status.idle":"2022-07-15T09:43:43.592977Z","shell.execute_reply.started":"2022-07-15T09:43:43.556042Z","shell.execute_reply":"2022-07-15T09:43:43.591811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[col_names] = numeric[col_names]\ntest[col_names_test] = numeric_test[col_names_test]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.594334Z","iopub.execute_input":"2022-07-15T09:43:43.595182Z","iopub.status.idle":"2022-07-15T09:43:43.614522Z","shell.execute_reply.started":"2022-07-15T09:43:43.595137Z","shell.execute_reply":"2022-07-15T09:43:43.613599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.615990Z","iopub.execute_input":"2022-07-15T09:43:43.616235Z","iopub.status.idle":"2022-07-15T09:43:43.641241Z","shell.execute_reply.started":"2022-07-15T09:43:43.616204Z","shell.execute_reply":"2022-07-15T09:43:43.639860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.643031Z","iopub.execute_input":"2022-07-15T09:43:43.643365Z","iopub.status.idle":"2022-07-15T09:43:43.669489Z","shell.execute_reply.started":"2022-07-15T09:43:43.643320Z","shell.execute_reply":"2022-07-15T09:43:43.668556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Duplicate Value\nprint(\"train: \\n\",train.loc[train.duplicated()].shape)\n#--------------------------------------------------\nprint(\"test: \\n\",test.loc[test.duplicated()].shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.671855Z","iopub.execute_input":"2022-07-15T09:43:43.672196Z","iopub.status.idle":"2022-07-15T09:43:43.715469Z","shell.execute_reply.started":"2022-07-15T09:43:43.672150Z","shell.execute_reply":"2022-07-15T09:43:43.714497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalization & Feature Selection","metadata":{}},{"cell_type":"code","source":"mask = np.triu(np.ones_like(train.corr()))\nfig, ax = plt.subplots(figsize=(150,120),dpi=80, facecolor='w', edgecolor='k')\nsns.heatmap(train.corr(), mask= mask, cmap=\"YlGnBu\", vmax=.2, annot = True, center = 0,\n            annot_kws={\"fontsize\":22})","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:43:43.716853Z","iopub.execute_input":"2022-07-15T09:43:43.717113Z","iopub.status.idle":"2022-07-15T09:44:06.110909Z","shell.execute_reply.started":"2022-07-15T09:43:43.717081Z","shell.execute_reply":"2022-07-15T09:44:06.109879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlation=(train.corr()['SalePrice'])\ndel correlation['SalePrice']\ncorrelation.sort_values(ascending=True).plot(kind='barh',color = \"darkgreen\", \n                                             figsize = (100,90), fontsize=50)\nplt.title(\"correlation\", fontsize=50)\nplt.axvline(x = 0.6, ymin = 0, ymax = 1,\n            linewidth = 5, linestyle =\"--\",\n            color ='#da1934');\nplt.axvline(x = -0.6, ymin = 0, ymax = 1,\n            linewidth = 5, linestyle =\"--\",\n            color ='#da1934');","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:06.112276Z","iopub.execute_input":"2022-07-15T09:44:06.112578Z","iopub.status.idle":"2022-07-15T09:44:09.839161Z","shell.execute_reply.started":"2022-07-15T09:44:06.112520Z","shell.execute_reply":"2022-07-15T09:44:09.838180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Highly correlated independent variables to target are : `ExterQual` , `BsmtQual`,`OverallQual`, `GrLivArea`, `GarageCars`, `GarageArea`, `TotalBsmtSF` and `1stFlrSF`.","metadata":{}},{"cell_type":"code","source":"new_train = train[['Id', 'ExterQual', 'BsmtQual', 'OverallQual', 'GrLivArea', 'GarageCars', \n                   'GarageArea', 'TotalBsmtSF', '1stFlrSF', 'SalePrice']]\n#------------------------------------------------------------------------------------------------\nnew_test = test[['Id', 'ExterQual', 'BsmtQual', 'OverallQual', 'GrLivArea', 'GarageCars', \n                   'GarageArea', 'TotalBsmtSF', '1stFlrSF']]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.840392Z","iopub.execute_input":"2022-07-15T09:44:09.841014Z","iopub.status.idle":"2022-07-15T09:44:09.848795Z","shell.execute_reply.started":"2022-07-15T09:44:09.840969Z","shell.execute_reply":"2022-07-15T09:44:09.847943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = new_train.drop(\"SalePrice\", axis = 1)\ny = new_train['SalePrice']","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.849901Z","iopub.execute_input":"2022-07-15T09:44:09.850114Z","iopub.status.idle":"2022-07-15T09:44:09.862577Z","shell.execute_reply.started":"2022-07-15T09:44:09.850088Z","shell.execute_reply":"2022-07-15T09:44:09.861806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(x,y, train_size = 0.7, test_size = 0.3, \n                                                  random_state = 100)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.863873Z","iopub.execute_input":"2022-07-15T09:44:09.864297Z","iopub.status.idle":"2022-07-15T09:44:09.878431Z","shell.execute_reply.started":"2022-07-15T09:44:09.864248Z","shell.execute_reply":"2022-07-15T09:44:09.877729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scale_features = MinMaxScaler()\ncol = ['ExterQual', 'BsmtQual', 'OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', \n       'TotalBsmtSF', '1stFlrSF']\n\nx_train[col] = scale_features.fit_transform(x_train[col])\nx_val[col] = scale_features.transform(x_val[col])\n#-----------------------------------------------------\nnew_test[col] = scale_features.transform(new_test[col])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.879968Z","iopub.execute_input":"2022-07-15T09:44:09.880387Z","iopub.status.idle":"2022-07-15T09:44:09.901008Z","shell.execute_reply.started":"2022-07-15T09:44:09.880333Z","shell.execute_reply":"2022-07-15T09:44:09.900016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scale_target = MinMaxScaler()\ny_train = pd.DataFrame(y_train)\ny_train = scale_target.fit_transform(y_train)\n\ny_val = pd.DataFrame(y_val)\ny_val = scale_target.transform(y_val)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.902271Z","iopub.execute_input":"2022-07-15T09:44:09.902502Z","iopub.status.idle":"2022-07-15T09:44:09.912517Z","shell.execute_reply.started":"2022-07-15T09:44:09.902468Z","shell.execute_reply":"2022-07-15T09:44:09.911795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_new = x_train[col]\nx_val_new = x_val[col]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.914016Z","iopub.execute_input":"2022-07-15T09:44:09.914471Z","iopub.status.idle":"2022-07-15T09:44:09.930838Z","shell.execute_reply.started":"2022-07-15T09:44:09.914438Z","shell.execute_reply":"2022-07-15T09:44:09.928667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_new, lm = build_model(x_train_new,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.932392Z","iopub.execute_input":"2022-07-15T09:44:09.932761Z","iopub.status.idle":"2022-07-15T09:44:09.969729Z","shell.execute_reply.started":"2022-07-15T09:44:09.932715Z","shell.execute_reply":"2022-07-15T09:44:09.968854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"greater_than_5_per = x_train_new.columns[lm.pvalues > 0.05]\n\ni = 1\nwhile(len(greater_than_5_per)!=0):\n    print(\"iteration {}\".format(i))\n    new_col = x_train_new.columns[lm.pvalues < 0.05]\n    print(\"num of new columns is {}\\nnew columns in iteration are\\n{}\".format(len(new_col),new_col))\n    x_train_new = x_train_new[new_col]\n    x_train_new, lm = build_model(x_train_new,y_train)\n    #----------------------------------------------------------\n    if 'const' in x_train_new.columns[lm.pvalues > 0.05]:\n        greater_than_5_per = x_train_new.columns[lm.pvalues > 0.05].drop('const')\n    i+=1\n    print(\"\\n\\n******************************************\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:09.970865Z","iopub.execute_input":"2022-07-15T09:44:09.971224Z","iopub.status.idle":"2022-07-15T09:44:10.004725Z","shell.execute_reply.started":"2022-07-15T09:44:09.971193Z","shell.execute_reply":"2022-07-15T09:44:10.003795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"font-family: Cursive; font-size:16px; background-color:#ecfff5; padding: 25px 20px\">\nValues of p-value are below 5% so we keep the attributes.\n</div>","metadata":{}},{"cell_type":"code","source":"checkVIF(x_train_new)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.006286Z","iopub.execute_input":"2022-07-15T09:44:10.006824Z","iopub.status.idle":"2022-07-15T09:44:10.029897Z","shell.execute_reply.started":"2022-07-15T09:44:10.006765Z","shell.execute_reply":"2022-07-15T09:44:10.028093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_new = x_train_new.drop('const',axis=1)\nnew_col = x_train_new.columns\nx_val_new = x_val[new_col]\nx_val_new.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.031977Z","iopub.execute_input":"2022-07-15T09:44:10.033144Z","iopub.status.idle":"2022-07-15T09:44:10.043257Z","shell.execute_reply.started":"2022-07-15T09:44:10.033092Z","shell.execute_reply":"2022-07-15T09:44:10.042108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models - `with tuned hyperparameters`","metadata":{}},{"cell_type":"code","source":"DT = DecisionTreeRegressor(criterion='friedman_mse', max_depth=6, min_samples_leaf=5,\n                      random_state=0)\nDT.fit(x_train_new, y_train)\n\ny_train_pred_DT = DT.predict(x_train_new)\ny_val_pred_DT = DT.predict(x_val_new)\n\nr2_score_DT_train = r2_score(y_train, y_train_pred_DT)\nr2_score_DT_val = r2_score(y_val, y_val_pred_DT)\nmae_DT = mean_absolute_error(y_val, y_val_pred_DT)\n\nprint('R2_score (train): ', r2_score_DT_train)\nprint('R2_score (test): ', r2_score_DT_val)\nprint(\"MAE: \", mae_DT)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.044749Z","iopub.execute_input":"2022-07-15T09:44:10.045667Z","iopub.status.idle":"2022-07-15T09:44:10.066228Z","shell.execute_reply.started":"2022-07-15T09:44:10.045618Z","shell.execute_reply":"2022-07-15T09:44:10.064659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR=LinearRegression()\nLR.fit(x_train_new, y_train)\n\ny_train_pred_LR = LR.predict(x_train_new)\ny_val_pred_LR = LR.predict(x_val_new)\n\nr2_score_LR_train = r2_score(y_train, y_train_pred_LR)\nr2_score_LR_val = r2_score(y_val, y_val_pred_LR)\nmae_LR = mean_absolute_error(y_val, y_val_pred_LR)\n\nprint('R2_score (train): ', r2_score_LR_train)\nprint('R2_score (test): ', r2_score_LR_val)\nprint(\"MAE: \", mae_LR)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.068123Z","iopub.execute_input":"2022-07-15T09:44:10.068709Z","iopub.status.idle":"2022-07-15T09:44:10.085624Z","shell.execute_reply.started":"2022-07-15T09:44:10.068659Z","shell.execute_reply":"2022-07-15T09:44:10.084646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Ri= Ridge(alpha=1, fit_intercept=False, solver='saga')\nRi.fit(x_train_new, y_train)\n\ny_train_pred_Ri = Ri.predict(x_train_new)\ny_val_pred_Ri = Ri.predict(x_val_new)\n\nr2_score_Ri_train = r2_score(y_train, y_train_pred_Ri)\nr2_score_Ri_val = r2_score(y_val, y_val_pred_Ri)\nmae_Ri = mean_absolute_error(y_val, y_val_pred_Ri)\n\nprint('R2_score (train): ', r2_score_Ri_train)\nprint('R2_score (test): ', r2_score_Ri_val)\nprint(\"MAE: \", mae_Ri)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.087152Z","iopub.execute_input":"2022-07-15T09:44:10.088286Z","iopub.status.idle":"2022-07-15T09:44:10.106396Z","shell.execute_reply.started":"2022-07-15T09:44:10.088230Z","shell.execute_reply":"2022-07-15T09:44:10.105574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EN= ElasticNet(alpha=0.001, fit_intercept=False, l1_ratio=0.0)\nEN.fit(x_train_new, y_train)\n\ny_train_pred_EN = EN.predict(x_train_new)\ny_val_pred_EN = EN.predict(x_val_new)\n\nr2_score_EN_train = r2_score(y_train, y_train_pred_EN)\nr2_score_EN_val = r2_score(y_val, y_val_pred_EN)\nmae_EN = mean_absolute_error(y_val, y_val_pred_EN)\n\nprint('R2_score (train): ', r2_score_EN_train)\nprint('R2_score (test): ', r2_score_EN_val)\nprint(\"MAE: \", mae_EN)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.107774Z","iopub.execute_input":"2022-07-15T09:44:10.108790Z","iopub.status.idle":"2022-07-15T09:44:10.137733Z","shell.execute_reply.started":"2022-07-15T09:44:10.108581Z","shell.execute_reply":"2022-07-15T09:44:10.136955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"KNN = KNeighborsRegressor()\nKNN.fit(x_train_new, y_train)\n\ny_train_pred_KNN = KNN.predict(x_train_new)\ny_val_pred_KNN = KNN.predict(x_val_new)\n\nr2_score_KNN_train = r2_score(y_train, y_train_pred_KNN)\nr2_score_KNN_val = r2_score(y_val, y_val_pred_KNN)\nmae_KNN = mean_absolute_error(y_val, y_val_pred_KNN)\n\nprint('R2_score (train): ', r2_score_KNN_train)\nprint('R2_score (test): ', r2_score_KNN_val)\nprint(\"MAE: \", mae_KNN)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.139018Z","iopub.execute_input":"2022-07-15T09:44:10.140051Z","iopub.status.idle":"2022-07-15T09:44:10.173247Z","shell.execute_reply.started":"2022-07-15T09:44:10.140000Z","shell.execute_reply":"2022-07-15T09:44:10.172374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RF = RandomForestRegressor(criterion='mae', min_samples_leaf=3, min_samples_split=5,\n                      n_estimators=200, random_state=0)\nRF.fit(x_train_new, y_train)\n\ny_train_pred_RF = RF.predict(x_train_new)\ny_val_pred_RF = RF.predict(x_val_new)\n\nr2_score_RF_train = r2_score(y_train, y_train_pred_RF)\nr2_score_RF_val = r2_score(y_val, y_val_pred_RF)\nmae_RF = mean_absolute_error(y_val, y_val_pred_RF)\n\nprint('R2_score (train): ', r2_score_RF_train)\nprint('R2_score (test): ', r2_score_RF_val)\nprint(\"MAE: \", mae_RF)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:10.174581Z","iopub.execute_input":"2022-07-15T09:44:10.174982Z","iopub.status.idle":"2022-07-15T09:44:13.653400Z","shell.execute_reply.started":"2022-07-15T09:44:10.174934Z","shell.execute_reply":"2022-07-15T09:44:13.652497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SVR_model = SVR(C=0.1, gamma=0.6)\nSVR_model.fit(x_train_new, y_train)\n\ny_train_pred_SVR_model = SVR_model.predict(x_train_new)\ny_val_pred_SVR_model = SVR_model.predict(x_val_new)\n\nr2_score_SVR_model_train = r2_score(y_train, y_train_pred_SVR_model)\nr2_score_SVR_model_val = r2_score(y_val, y_val_pred_SVR_model)\nmae_SVR_model = mean_absolute_error(y_val, y_val_pred_SVR_model)\n\nprint('R2_score (train): ', r2_score_SVR_model_train)\nprint('R2_score (test): ', r2_score_SVR_model_val)\nprint(\"MAE: \", mae_SVR_model)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:13.654807Z","iopub.execute_input":"2022-07-15T09:44:13.655261Z","iopub.status.idle":"2022-07-15T09:44:13.686269Z","shell.execute_reply.started":"2022-07-15T09:44:13.655229Z","shell.execute_reply":"2022-07-15T09:44:13.685540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ET = ExtraTreesRegressor(min_samples_leaf=4, min_samples_split=5, n_estimators=50,\n                    random_state=0)\nET.fit(x_train_new, y_train)\n\ny_train_pred_ET = ET.predict(x_train_new)\ny_val_pred_ET = ET.predict(x_val_new)\n\nr2_score_ET_train = r2_score(y_train, y_train_pred_ET)\nr2_score_ET_val = r2_score(y_val, y_val_pred_ET)\nmae_ET = mean_absolute_error(y_val, y_val_pred_ET)\n\nprint('R2_score (train): ', r2_score_ET_train)\nprint('R2_score (test): ', r2_score_ET_val)\nprint(\"MAE: \", mae_ET)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:13.687451Z","iopub.execute_input":"2022-07-15T09:44:13.687701Z","iopub.status.idle":"2022-07-15T09:44:13.808424Z","shell.execute_reply.started":"2022-07-15T09:44:13.687672Z","shell.execute_reply":"2022-07-15T09:44:13.807290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GBR = GradientBoostingRegressor(min_samples_leaf=4, min_samples_split=5,\n                          n_estimators=50, random_state=0)\nGBR.fit(x_train_new, y_train)\n\ny_train_pred_GBR = GBR.predict(x_train_new)\ny_val_pred_GBR = GBR.predict(x_val_new)\n\nr2_score_GBR_train = r2_score(y_train, y_train_pred_GBR)\nr2_score_GBR_val = r2_score(y_val, y_val_pred_GBR)\nmae_GBR = mean_absolute_error(y_val, y_val_pred_GBR)\n\nprint('R2_score (train): ', r2_score_GBR_train)\nprint('R2_score (test): ', r2_score_GBR_val)\nprint(\"MAE: \", mae_GBR)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:13.810035Z","iopub.execute_input":"2022-07-15T09:44:13.810329Z","iopub.status.idle":"2022-07-15T09:44:13.893352Z","shell.execute_reply.started":"2022-07-15T09:44:13.810289Z","shell.execute_reply":"2022-07-15T09:44:13.892486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XGBR = XGBRegressor(n_estimators = 50, max_depth=3, learning_rate_init=0.2)\nXGBR.fit(x_train_new, y_train)\n\ny_train_pred_XGBR = XGBR.predict(x_train_new)\ny_val_pred_XGBR = XGBR.predict(x_val_new)\n\nr2_score_XGBR_train = r2_score(y_train, y_train_pred_XGBR)\nr2_score_XGBR_val = r2_score(y_val, y_val_pred_XGBR)\nmae_XGBR = mean_absolute_error(y_val, y_val_pred_XGBR)\n\nprint('R2_score (train): ', r2_score_XGBR_train)\nprint('R2_score (test): ', r2_score_XGBR_val)\nprint(\"MAE: \", mae_XGBR)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:13.894887Z","iopub.execute_input":"2022-07-15T09:44:13.895135Z","iopub.status.idle":"2022-07-15T09:44:14.086070Z","shell.execute_reply.started":"2022-07-15T09:44:13.895106Z","shell.execute_reply":"2022-07-15T09:44:14.085394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare Models","metadata":{}},{"cell_type":"code","source":"models = [('Decision Tree Regressor', mae_DT, r2_score_DT_train, r2_score_DT_val,''),\n          ('Linear Regression', mae_LR, r2_score_LR_train, r2_score_LR_val,'Stable model-Based on R2'),\n          ('Ridge', mae_Ri, r2_score_Ri_train, r2_score_Ri_val,'Stable model-Based on R2'),\n          ('ElasticNet', mae_EN, r2_score_EN_train, r2_score_EN_val,'Stable model-Based on R2'),\n          ('KNeighbors Regressor', mae_KNN, r2_score_KNN_train, r2_score_KNN_val,''),\n          ('Random Forest Regressor', mae_RF, r2_score_RF_train, r2_score_RF_val,''),\n          ('SVR', mae_SVR_model, r2_score_SVR_model_train, r2_score_SVR_model_val,'Stable model-Based on R2'),\n          ('Extra Trees Regressor', mae_ET, r2_score_ET_train, r2_score_ET_val,'The first best-Based on MAE'),\n          ('Gradient Boosting Regressor', mae_GBR, r2_score_GBR_train, r2_score_GBR_val,'The second best-Based on MAE'),\n          ('XGBRegressor', mae_XGBR, r2_score_XGBR_train, r2_score_XGBR_val,'')]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.087264Z","iopub.execute_input":"2022-07-15T09:44:14.087921Z","iopub.status.idle":"2022-07-15T09:44:14.095221Z","shell.execute_reply.started":"2022-07-15T09:44:14.087883Z","shell.execute_reply":"2022-07-15T09:44:14.094546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compare_models  = pd.DataFrame(data = models, columns=['Model', 'MAE', 'R2_Score(train)', 'R2_Score(test)', 'Description'])\ncompare_models.style.background_gradient(cmap='YlGn')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.096445Z","iopub.execute_input":"2022-07-15T09:44:14.096847Z","iopub.status.idle":"2022-07-15T09:44:14.189269Z","shell.execute_reply.started":"2022-07-15T09:44:14.096815Z","shell.execute_reply":"2022-07-15T09:44:14.188263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\ncompare_models.sort_values(by=['MAE'], ascending=True, inplace=True)\nsns.barplot(x='MAE', y='Model', data = compare_models, palette='plasma')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.190813Z","iopub.execute_input":"2022-07-15T09:44:14.191125Z","iopub.status.idle":"2022-07-15T09:44:14.551587Z","shell.execute_reply.started":"2022-07-15T09:44:14.191083Z","shell.execute_reply":"2022-07-15T09:44:14.550743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Model","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=[15,5])\n\nc = [i for i in range(1,439,1)]\nd = [i for i in range(1,439,1)]\n\nplt.plot(c, y_val, color=\"blue\", linewidth=1, linestyle=\"-\")\nplt.plot(d, y_val_pred_ET, color=\"red\",  linewidth=1, linestyle=\"-\")\nplt.xlabel('Index', fontsize=18) \nplt.ylabel('SalePrice', fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.553030Z","iopub.execute_input":"2022-07-15T09:44:14.553286Z","iopub.status.idle":"2022-07-15T09:44:14.829085Z","shell.execute_reply.started":"2022-07-15T09:44:14.553255Z","shell.execute_reply":"2022-07-15T09:44:14.828395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val = y_val.ravel()\nactualvspredicted = pd.DataFrame({\"Actual\":y_val,\"Predicted\":y_val_pred_ET, 'Different':y_val-y_val_pred_ET})\nactualvspredicted.head(10).style.background_gradient(cmap='YlGn')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.830036Z","iopub.execute_input":"2022-07-15T09:44:14.830916Z","iopub.status.idle":"2022-07-15T09:44:14.852121Z","shell.execute_reply.started":"2022-07-15T09:44:14.830878Z","shell.execute_reply":"2022-07-15T09:44:14.851200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=[8,6])\np = sns.jointplot(actualvspredicted['Predicted'],actualvspredicted['Actual'], kind='reg')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:14.853614Z","iopub.execute_input":"2022-07-15T09:44:14.853861Z","iopub.status.idle":"2022-07-15T09:44:15.578637Z","shell.execute_reply.started":"2022-07-15T09:44:14.853831Z","shell.execute_reply":"2022-07-15T09:44:15.577869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"new_test.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:15.579723Z","iopub.execute_input":"2022-07-15T09:44:15.580215Z","iopub.status.idle":"2022-07-15T09:44:15.586666Z","shell.execute_reply.started":"2022-07-15T09:44:15.580169Z","shell.execute_reply":"2022-07-15T09:44:15.585830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_feature = new_test[new_col]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:15.588307Z","iopub.execute_input":"2022-07-15T09:44:15.588638Z","iopub.status.idle":"2022-07-15T09:44:15.599410Z","shell.execute_reply.started":"2022-07-15T09:44:15.588594Z","shell.execute_reply":"2022-07-15T09:44:15.598783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = ET.predict(test_final_feature)\npredictions = predictions.reshape(-1,1)\n\ninverse_pred = scale_target.inverse_transform(predictions)\ninverse_pred = inverse_pred.ravel()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:15.600987Z","iopub.execute_input":"2022-07-15T09:44:15.601675Z","iopub.status.idle":"2022-07-15T09:44:15.625402Z","shell.execute_reply.started":"2022-07-15T09:44:15.601627Z","shell.execute_reply":"2022-07-15T09:44:15.624548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame({'Id': new_test.Id, 'SalePrice': inverse_pred})\noutput.to_csv('my_submission.csv', index=False)\nprint(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:44:15.626703Z","iopub.execute_input":"2022-07-15T09:44:15.627803Z","iopub.status.idle":"2022-07-15T09:44:15.642433Z","shell.execute_reply.started":"2022-07-15T09:44:15.627762Z","shell.execute_reply":"2022-07-15T09:44:15.641786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"border-radius:10px;\n            background-color:#ffffff;\n            border-style: solid;\n            border-color: #015a2c;\n            letter-spacing:0.5px;\">\n\n<center><h4 style=\"padding: 5px 0px; color:#015a2c; font-weight: bold; font-family: Cursive\">\n    Thanks for your attention and for reviewing my notebook.🙌 <br><br>Please write your comments for me.📝</h4></center>\n<center><h4 style=\"padding: 5px 0px; color:#015a2c; font-weight: bold; font-family: Cursive\">\nIf you liked my work and found it useful, please upvote. Thank you🙏</h4></center>\n</div>","metadata":{}}]}