{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "2708147a-4e50-5a06-d5a4-3b17375e657d" }, "source": [ "For variables retreatement, we used Borys Klyus' approach\n", "https://www.kaggle.com/klyusba/house-prices-advanced-regression-techniques/lasso-model-for-regression-problem/run/368430 \n", "\n", "For modeling, we used Alexandru Papiu's notebook\n", "https://www.kaggle.com/apapiu/house-prices-advanced-regression-techniques/regularized-linear-models/run/573627" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6c1b5539-1759-e753-6dae-41f655ad31eb" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "from sklearn.linear_model import Lasso\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "from scipy.stats.stats import pearsonr\n", "\n", "\n", "%config InlineBackend.figure_format = 'retina' #set 'png' here when working on notebook\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4966d147-a568-0ce2-d84e-6dd84b2c62eb" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e63b776d-c45f-1ec3-cfbb-8676b1f907ac" }, "outputs": [], "source": [ "train.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a8af6127-0180-983a-8ffd-ec18d569bbb8" }, "outputs": [], "source": [ "all_data = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'],\n", " test.loc[:,'MSSubClass':'SaleCondition']), ignore_index=True)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7ffbc4c8-bf8e-b7ad-8a31-7cd41bfbb1fa" }, "source": [ "## Data preprocessing ##\n", "\n", "1. Replace the numeric missing values (NaN's) with the most common values, replace qualitative feathers by reasonable ordered numeric values, convert similar feathers into price categories, weaken less significant modalities\u2019 impact, process outliers, and so on.\n", "2. Log - Transformation \n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41f21588-54a2-cccf-ebfb-b82f22818d08" }, "outputs": [], "source": [ "x = all_data.loc[np.logical_not(all_data[\"LotFrontage\"].isnull()), \"LotArea\"]\n", "y = all_data.loc[np.logical_not(all_data[\"LotFrontage\"].isnull()), \"LotFrontage\"]\n", "# plt.scatter(x, y)\n", "t = (x <= 25000) & (y <= 150)\n", "p = np.polyfit(x[t], y[t], 1)\n", "all_data.loc[all_data['LotFrontage'].isnull(), 'LotFrontage'] = np.polyval(p, all_data.loc[all_data['LotFrontage'].isnull(), 'LotArea'])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "adad1d8c-1f3a-cc8d-eebd-d0f97110a7da" }, "source": [ "There are many features were NaN should be considered as absence of such property. In other cases I replace NaN with most common value" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3bfae950-1716-ccdd-1fe7-c01a0f07cfd3" }, "outputs": [], "source": [ "all_data.loc[all_data.Alley.isnull(), 'Alley'] = 'NoAlley'\n", "all_data.loc[all_data.MasVnrType.isnull(), 'MasVnrType'] = 'None' # no good\n", "all_data.loc[all_data.MasVnrType == 'None', 'MasVnrArea'] = 0\n", "all_data.loc[all_data.BsmtQual.isnull(), 'BsmtQual'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtCond.isnull(), 'BsmtCond'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtExposure.isnull(), 'BsmtExposure'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType1.isnull(), 'BsmtFinType1'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType1=='NoBsmt', 'BsmtFinSF1'] = 0\n", "all_data.loc[all_data.BsmtFinType2=='NoBsmt', 'BsmtFinSF2'] = 0\n", "all_data.loc[all_data.BsmtFinSF1.isnull(), 'BsmtFinSF1'] = all_data.BsmtFinSF1.median()\n", "all_data.loc[all_data.BsmtQual=='NoBsmt', 'BsmtUnfSF'] = 0\n", "all_data.loc[all_data.BsmtUnfSF.isnull(), 'BsmtUnfSF'] = all_data.BsmtUnfSF.median()\n", "all_data.loc[all_data.BsmtQual=='NoBsmt', 'TotalBsmtSF'] = 0\n", "all_data.loc[all_data.FireplaceQu.isnull(), 'FireplaceQu'] = 'NoFireplace'\n", "all_data.loc[all_data.GarageType.isnull(), 'GarageType'] = 'NoGarage'\n", "all_data.loc[all_data.GarageFinish.isnull(), 'GarageFinish'] = 'NoGarage'\n", "all_data.loc[all_data.GarageQual.isnull(), 'GarageQual'] = 'NoGarage'\n", "all_data.loc[all_data.GarageCond.isnull(), 'GarageCond'] = 'NoGarage'\n", "all_data.loc[all_data.BsmtFullBath.isnull(), 'BsmtFullBath'] = 0\n", "all_data.loc[all_data.BsmtHalfBath.isnull(), 'BsmtHalfBath'] = 0\n", "all_data.loc[all_data.KitchenQual.isnull(), 'KitchenQual'] = 'TA'\n", "all_data.loc[all_data.MSZoning.isnull(), 'MSZoning'] = 'RL'\n", "all_data.loc[all_data.Utilities.isnull(), 'Utilities'] = 'AllPub'\n", "all_data.loc[all_data.Exterior1st.isnull(), 'Exterior1st'] = 'VinylSd'\n", "all_data.loc[all_data.Exterior2nd.isnull(), 'Exterior2nd'] = 'VinylSd'\n", "all_data.loc[all_data.Functional.isnull(), 'Functional'] = 'Typ'\n", "all_data.loc[all_data.SaleCondition.isnull(), 'SaleCondition'] = 'Normal'\n", "all_data.loc[all_data.SaleCondition.isnull(), 'SaleType'] = 'WD'\n", "all_data.loc[all_data['PoolQC'].isnull(), 'PoolQC'] = 'NoPool'\n", "all_data.loc[all_data['Fence'].isnull(), 'Fence'] = 'NoFence'\n", "all_data.loc[all_data['MiscFeature'].isnull(), 'MiscFeature'] = 'None'\n", "all_data.loc[all_data['Electrical'].isnull(), 'Electrical'] = 'SBrkr'\n", "# only one is null and it has type Detchd\n", "all_data.loc[all_data['GarageArea'].isnull(), 'GarageArea'] = all_data.loc[all_data['GarageType']=='Detchd', 'GarageArea'].mean()\n", "all_data.loc[all_data['GarageCars'].isnull(), 'GarageCars'] = all_data.loc[all_data['GarageType']=='Detchd', 'GarageCars'].median()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f706c097-313b-d53e-908d-2d02a10868c4" }, "outputs": [], "source": [ "\n", "all_data = all_data.replace({'Utilities': {'AllPub': 1, 'NoSeWa': 0, 'NoSewr': 0, 'ELO': 0},\n", " 'Street': {'Pave': 1, 'Grvl': 0 },\n", " 'FireplaceQu': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoFireplace': 0 \n", " },\n", " 'Fence': {'GdPrv': 2, \n", " 'GdWo': 2, \n", " 'MnPrv': 1, \n", " 'MnWw': 1,\n", " 'NoFence': 0},\n", " 'ExterQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1\n", " },\n", " 'ExterCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1\n", " },\n", " 'BsmtQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoBsmt': 0},\n", " 'BsmtExposure': {'Gd': 3, \n", " 'Av': 2, \n", " 'Mn': 1,\n", " 'No': 0,\n", " 'NoBsmt': 0},\n", " 'BsmtCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoBsmt': 0},\n", " 'GarageQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoGarage': 0},\n", " 'GarageCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoGarage': 0},\n", " 'KitchenQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1},\n", " 'Functional': {'Typ': 0,\n", " 'Min1': 1,\n", " 'Min2': 1,\n", " 'Mod': 2,\n", " 'Maj1': 3,\n", " 'Maj2': 4,\n", " 'Sev': 5,\n", " 'Sal': 6} \n", " })" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a22c83af-1b99-80c5-c9e0-ae3b98fc8277" }, "outputs": [], "source": [ "newer_dwelling = all_data.MSSubClass.replace({20: 1, \n", " 30: 0, \n", " 40: 0, \n", " 45: 0,\n", " 50: 0, \n", " 60: 1,\n", " 70: 0,\n", " 75: 0,\n", " 80: 0,\n", " 85: 0,\n", " 90: 0,\n", " 120: 1,\n", " 150: 0,\n", " 160: 0,\n", " 180: 0,\n", " 190: 0})\n", "newer_dwelling.name = 'newer_dwelling'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fcc7f095-ba43-0377-66bc-8ebdfbb18e31" }, "outputs": [], "source": [ "all_data = all_data.replace({'MSSubClass': {20: 'SubClass_20', \n", " 30: 'SubClass_30', \n", " 40: 'SubClass_40', \n", " 45: 'SubClass_45',\n", " 50: 'SubClass_50', \n", " 60: 'SubClass_60',\n", " 70: 'SubClass_70',\n", " 75: 'SubClass_75',\n", " 80: 'SubClass_80',\n", " 85: 'SubClass_85',\n", " 90: 'SubClass_90',\n", " 120: 'SubClass_120',\n", " 150: 'SubClass_150',\n", " 160: 'SubClass_160',\n", " 180: 'SubClass_180',\n", " 190: 'SubClass_190'}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4245cfd6-86f6-4590-fbf3-3f292f3f0495" }, "outputs": [], "source": [ "# The idea is good quality should rise price, poor quality - reduce price\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "qu_list = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "\n", "bad_heating = all_data.HeatingQC.replace({'Ex': 0, \n", " 'Gd': 0, \n", " 'TA': 0, \n", " 'Fa': 1,\n", " 'Po': 1})\n", "bad_heating.name = 'bad_heating'\n", " \n", "MasVnrType_Any = all_data.MasVnrType.replace({'BrkCmn': 1,\n", " 'BrkFace': 1,\n", " 'CBlock': 1,\n", " 'Stone': 1,\n", " 'None': 0})\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace({'Abnorml': 1,\n", " 'Alloca': 1,\n", " 'AdjLand': 1,\n", " 'Family': 1,\n", " 'Normal': 0,\n", " 'Partial': 0})\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "Neighborhood_Good = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Neighborhood_Good'])\n", "Neighborhood_Good[all_data.Neighborhood=='NridgHt'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='Crawfor'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='StoneBr'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='Somerst'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='NoRidge'] = 1" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8cbb3844-4bef-bbad-3d78-6f834300d110" }, "source": [ "I have no idea what to do with Exterior1st, Exterior2nd, RoofMatl, Condition1, Condition2, BldgType. I'll try convert them into some kind of price brackets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f0fdf976-c5bb-fe0b-3ffe-511197bf11b4" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc], sparse=True)\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b7ecdeae-937b-a58d-befc-4d2b638e1041" }, "outputs": [], "source": [ "p = train.SalePrice/100000\n", "plt.hist(p[pc_pred=='pc1'])\n", "plt.hist(p[pc_pred=='pc2'])\n", "plt.hist(p[pc_pred=='pc3'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b444fe09-fd4b-28e9-a08d-6080f8e9ea1d" }, "outputs": [], "source": [ "price_category = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['pc'])\n", "X_t = pd.get_dummies(all_data.loc[:, columns_for_pc], sparse=True)\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "price_category = price_category.to_sparse()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df571648-bb68-1eed-ad9f-e924911f4dbf" }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "season = all_data.MoSold.replace( {1: 0, \n", " 2: 0, \n", " 3: 0, \n", " 4: 1,\n", " 5: 1, \n", " 6: 1,\n", " 7: 1,\n", " 8: 0,\n", " 9: 0,\n", " 10: 0,\n", " 11: 0,\n", " 12: 0})\n", "season.name = 'season'\n", "\n", "# Numer month is not significant\n", "all_data = all_data.replace({'MoSold': {1: 'Yan', \n", " 2: 'Feb', \n", " 3: 'Mar', \n", " 4: 'Apr',\n", " 5: 'May', \n", " 6: 'Jun',\n", " 7: 'Jul',\n", " 8: 'Avg',\n", " 9: 'Sep',\n", " 10: 'Oct',\n", " 11: 'Nov',\n", " 12: 'Dec'}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7428a0fe-0695-1709-fbee-e57148f7eb39" }, "outputs": [], "source": [ "all_data = all_data.replace({'CentralAir': {'Y': 1, \n", " 'N': 0}})\n", "all_data = all_data.replace({'PavedDrive': {'Y': 1, \n", " 'P': 0,\n", " 'N': 0}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4a4b1ece-a80f-0071-8020-522e9c75fa26" }, "outputs": [], "source": [ "reconstruct = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Reconstruct'])\n", "reconstruct[all_data.YrSold < all_data.YearRemodAdd] = 1\n", "reconstruct = reconstruct.to_sparse()\n", "\n", "recon_after_buy = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['ReconstructAfterBuy'])\n", "recon_after_buy[all_data.YearRemodAdd >= all_data.YrSold] = 1\n", "recon_after_buy = recon_after_buy.to_sparse()\n", "\n", "build_eq_buy = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Build.eq.Buy'])\n", "build_eq_buy[all_data.YearBuilt >= all_data.YrSold] = 1\n", "build_eq_buy = build_eq_buy.to_sparse()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4aac1977-9a4c-0dfe-8abc-737cb0c945cb" }, "outputs": [], "source": [ "\n", "all_data.YrSold = 2010 - all_data.YrSold" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "54ee71df-fdcd-00df-6adf-f76453daacdf" }, "outputs": [], "source": [ "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "all_data.GarageYrBlt = all_data.GarageYrBlt.map(year_map)\n", "all_data.loc[all_data['GarageYrBlt'].isnull(), 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "060e746d-16fb-1d41-468a-584b1c5b1cb3" }, "outputs": [], "source": [ "all_data.YearBuilt = all_data.YearBuilt.map(year_map)\n", "all_data.YearRemodAdd = all_data.YearRemodAdd.map(year_map)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6ebb4194-80fb-a91d-e153-cd38688b4a1d" }, "source": [ "Scaling numeric data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2b153f12-8081-ebe4-e503-3c55c8350618" }, "outputs": [], "source": [ "numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n", "\n", "t = all_data[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "all_data[use_max_scater] = all_data[use_max_scater]/all_data[use_max_scater].max()\n", "all_data[use_95_scater] = all_data[use_95_scater]/all_data[use_95_scater].quantile(.95)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "89aace43-ac8c-3c7d-eaf7-c6d85adec796" }, "outputs": [], "source": [ "t = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', \n", " '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', \n", " 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']\n", "\n", "all_data.loc[:, t] = np.log1p(all_data.loc[:, t])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "063dfaf9-7f49-5c62-655b-8c35d5d2f19e" }, "source": [ "## Preparing for sklearn ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "83da8155-1385-dce5-9e8b-4630ef12ff61" }, "outputs": [], "source": [ "# all classes in sklearn requires numeric data only\n", "# transform categorical variable into binary\n", "X = pd.get_dummies(all_data, sparse=True)\n", "X = X.fillna(0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a64a33b1-9cd0-986a-bc12-76b30a69f912" }, "outputs": [], "source": [ "X = X.drop('RoofMatl_ClyTile', axis=1) # only one is not zero\n", "X = X.drop('Condition2_PosN', axis=1) # only two is not zero\n", "X = X.drop('MSZoning_C (all)', axis=1)\n", "X = X.drop('MSSubClass_SubClass_160', axis=1)\n", "# this features definitely couse overfitting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c3e142c-018d-484d-1b49-997d671a8b12" }, "outputs": [], "source": [ "# add new features\n", "X = pd.concat((X, newer_dwelling, season, reconstruct, recon_after_buy,\n", " qu_list, bad_heating, MasVnrType_Any, price_category, build_eq_buy), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a3d8ee03-439b-1fe1-0170-2948e08f2fb2" }, "source": [ "Next step is guess what new feachers we need to intoduse to make the model better. I'll make a lot of feachers and model will choose" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ab19a761-2876-11c8-3fdb-9fbe703872e3" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " # t = [s for s in X.axes[1].get_values() if s not in areas]\n", " t = chain(qu_list.axes[1].get_values(), \n", " ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtCond', 'GarageQual', 'GarageCond',\n", " 'KitchenQual', 'HeatingQC', 'bad_heating', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Reconstruct',\n", " 'ReconstructAfterBuy', 'Build.eq.Buy'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(X), axis=1)\n", "X = pd.concat((X, XP), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cbdd91b4-6ea3-d1bd-3f13-8b92f9265048" }, "outputs": [], "source": [ "X_train = X[:train.shape[0]]\n", "X_test = X[train.shape[0]:]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b4560804-7a54-6f1f-6781-1b173b55e5cd" }, "outputs": [], "source": [ "# the model has become really big\n", "X_train.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b9faea74-013d-0813-9715-44d11b6665ff" }, "outputs": [], "source": [ "y = np.log1p(train.SalePrice)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4be5d5c6-91c1-3f5f-c548-82014a276284" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7b9a32b8-a62d-08c0-523c-d70b8dd23140" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id)\n", "y = y.drop(outliers_id)\n", "# There are difinetly more outliers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fff9d4fd-4ef4-312e-6710-43a7d2a9389b" }, "outputs": [], "source": [ "from sklearn.linear_model import Ridge, RidgeCV, LassoCV, LassoLarsCV\n", "from sklearn.model_selection import cross_val_score\n", "\n", "def rmse_cv(model):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring=\"neg_mean_squared_error\", cv = 5))\n", " return(rmse)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "85c9c2d0-3161-2a4f-da46-55cdb3b72059" }, "source": [ "Less folds number at the cross-validation stage makes a worse performance, it\u2019s appropriate to keep cv = 5." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8a3a71df-77eb-e65f-0bc3-180bb92f6022" }, "source": [ "## Model ##" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8a0d1ec7-d9cd-f1b0-5558-d6039636adaf" }, "source": [ "**Ridge** - Linear least squares with l2 regularization.\n", "\n", "The main tuning parameter for the Ridge model is alpha - a regularization parameter that measures how flexible our model is. It reduces the variance of the estimates. The higher the regularization the less prone our model will be to overfit. However it will also lose flexibility and might not capture all of the signal in the data.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dcbc8cb0-acc2-2662-514b-fe54ec589bbe" }, "outputs": [], "source": [ "alphas_ridge = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]\n", "cv_rmse_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() \n", " for alpha in alphas_ridge]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b106d046-ad35-c784-62f5-7f963281678d" }, "outputs": [], "source": [ "print (cv_rmse_ridge)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ac9de04a-a9e6-cee7-5e11-45a29850ca52" }, "outputs": [], "source": [ "cv_ridge = pd.Series(cv_rmse_ridge, index = alphas_ridge)\n", "cv_ridge.plot(title = \"Validation Ridge\")\n", "plt.xlabel(\"alphas\")\n", "plt.ylabel(\"rmse\")" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e679fc70-4e24-27f5-de3c-97ad1108e04d" }, "source": [ "Note the U-ish shaped curve above. When alpha is too large the regularization is too strong and the model cannot capture all the complexities in the data. If however we let the model be too flexible (alpha small) the model begins to overfit. A value of alpha = 10 is about right based on the plot above." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "07636083-ca2e-e860-ea46-6f0988720791" }, "outputs": [], "source": [ "model_ridge = Ridge(alpha = 10).fit(X_train, y)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e3d7b4ac-189e-2632-d39f-4c2b1654d4cb" }, "outputs": [], "source": [ "cv_ridge.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f31cfd0c-71b3-7067-13db-1e85199f157a" }, "outputs": [], "source": [ "#let's look at the residuals as well:\n", "matplotlib.rcParams['figure.figsize'] = (6.0, 6.0)\n", "\n", "preds_ridge = pd.DataFrame({\"preds Ridge\":model_ridge.predict(X_train), \"true\":y})\n", "preds_ridge[\"residuals\"] = preds_ridge[\"true\"] - preds_ridge[\"preds Ridge\"]\n", "preds_ridge.plot(x = \"preds Ridge\", y = \"residuals\",kind = \"scatter\")" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "5d6ab27c-40c4-c901-29e8-d254db19ff26" }, "source": [ "**Lasso** L1 prior as regularizer\n", "\n", "Let' try out the Lasso model. We will do a slightly different approach here and use the built in Lasso CV to figure out the best alpha for us. For some reason the alphas in Lasso CV are really the inverse or the alphas in Ridge. Alpha : mutliplies the L1 term." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b3313ba6-b46d-c4d5-67af-3208966dc564" }, "outputs": [], "source": [ "model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005]).fit(X_train, y)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0e4ba5b2-1f81-38d3-42cb-03942cf6542a" }, "outputs": [], "source": [ "cv_rmse_lasso = rmse_cv(model_lasso).mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a183f839-7605-1869-fefe-1fcd8b5b77fe" }, "outputs": [], "source": [ "print (cv_rmse_lasso)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b57b860b-6b65-3a97-01d9-912a955c1c99" }, "source": [ "The Lasso regression gives us a less rmse, which means it performs better than a ridge regression. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8b9ba96d-6d1c-d6dc-6a7b-471c5ecabef9" }, "outputs": [], "source": [ "#let's look at the residuals as well:\n", "matplotlib.rcParams['figure.figsize'] = (6.0, 6.0)\n", "\n", "preds_lasso= pd.DataFrame({\"preds Lasso\":model_lasso.predict(X_train), \"true\":y})\n", "preds_lasso[\"residuals\"] = preds_lasso[\"true\"] - preds_lasso[\"preds Lasso\"]\n", "preds_lasso.plot(x = \"preds Lasso\", y = \"residuals\",kind = \"scatter\")" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7f8bb961-b6c9-c333-97cb-52c93ce6a6cf" }, "source": [ "The residual plot looks pretty good. So Lasso regression might be a good alternative to predict the price, it works well even without any combination" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c72fa4b8-8fd7-44ed-1bb8-b1a4721af464" }, "source": [ "**xgboost model**\n", "\n", "We could also add an xgboost model to the linear model to see if we can improve our score. Alexandru Papiu has used a max-depth which equals to 2. We thought this was not sufficient to learn, so we fixed it at 6." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "13856ef2-5d71-68b0-9e6c-832ff8b25f48" }, "outputs": [], "source": [ "import xgboost as xgb\n", "dtrain = xgb.DMatrix(X_train, label = y)\n", "dtest = xgb.DMatrix(X_test)\n", "\n", "params = {\"max_depth\":6, \"eta\":0.1}\n", "model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "740ba958-829a-b079-e525-daf19d1c1f52" }, "outputs": [], "source": [ "model.loc[30:,[\"test-rmse-mean\", \"train-rmse-mean\"]].plot()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b446175f-96a6-32fb-3753-7352dbc0c676" }, "outputs": [], "source": [ "model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=6, learning_rate=0.1) #the params were tuned using xgb.cv\n", "model_xgb.fit(X_train, y)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f4de178d-ef5a-6e53-35d8-aa8ce6d40db5" }, "source": [ "**Prediction**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80753de6-b27f-ac21-45cf-5c1405aea617" }, "outputs": [], "source": [ "lasso_preds = np.expm1(model_lasso.predict(X_test))\n", "ridge_preds = np.expm1(model_ridge.predict(X_test))\n", "xgb_preds = np.expm1(model_xgb.predict(X_test))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25c25103-7a20-d89b-1423-be6b74ba251a" }, "outputs": [], "source": [ "preds = 0.65*lasso_preds + 0.15*ridge_preds + 0.2*xgb_preds" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "21c37c17-5ae2-1aff-ba76-898ceaea926c" }, "outputs": [], "source": [ "from mpl_toolkits.mplot3d import Axes3D\n", "import matplotlib.pyplot as plt\n", "import statsmodels.formula.api as sm\n", "from matplotlib import cm" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b9fcda46-4f7d-6359-7a3a-1a5df6504b73" }, "outputs": [], "source": [ "predictions = pd.DataFrame({\"lasso\":lasso_preds, \"ridge\":ridge_preds, \"xgb\": xgb_preds})\n", "fig = plt.figure()\n", "ax = fig.add_subplot(111, projection='3d')\n", "\n", "x_surf = np.arange(0, 350, 20) # generate a mesh\n", "y_surf = np.arange(0, 60, 4)\n", "x_surf, y_surf = np.meshgrid(x_surf, y_surf)\n", "\n", "ax.plot_surface(lasso_preds,ridge_preds,\n", " xgb_preds.reshape(lasso_preds.shape),\n", " rstride=1,\n", " cstride=1,\n", " color='None',\n", " alpha = 0.4)\n", "\n", "ax.scatter(predictions['lasso'], predictions['ridge'], predictions['xgb'],\n", " c='red',\n", " marker='.',\n", " alpha=1)\n", "\n", "\n", "ax.set_xlabel('lasso')\n", "ax.set_ylabel('ridge')\n", "ax.set_zlabel('xgb')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2df5fee4-9af0-a3be-d5a0-40dbac81f375" }, "outputs": [], "source": [ "score = [0.12087,0.14608,0.12214,0.12656,0.12097,0.12016,0.12578,0.13411,0.12095,0.11996,0.12144,0.12012,0.11977,0.12042,0.12,0.12066,0.12068,0.16514,0.12099,0.12159,0.12095,0.12081,0.12088,0.11977,0.11968,0.11979,0.11995,0.1216,0.12097,0.11766,0.11948,0.11543]\n", "\n", "plt.title(\"Our score\")\n", "plt.plot(range(0,len(score)), score)\n", "plt.xlabel('Submission')\n", "plt.ylabel('Score')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "39a9eeb6-2668-0e4c-d429-6d5237303563" }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\":test.Id, \"SalePrice\":preds})\n", "solution.to_csv(\"House_Price_final_12032017.csv\", index = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "67a223b5-ccf4-7fda-c6d3-56f24fe24f78" }, "outputs": [], "source": [ "" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }