{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "2708147a-4e50-5a06-d5a4-3b17375e657d" }, "source": [ "For variables retreatement, we used Borys Klyus' approach\n", "https://www.kaggle.com/klyusba/house-prices-advanced-regression-techniques/lasso-model-for-regression-problem/run/368430 \n", "\n", "For modeling, we used Alexandru Papiu's notebook\n", "https://www.kaggle.com/apapiu/house-prices-advanced-regression-techniques/regularized-linear-models/run/573627" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6c1b5539-1759-e753-6dae-41f655ad31eb" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib\n", "import matplotlib.pyplot as plt\n", "from sklearn.linear_model import Lasso\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4966d147-a568-0ce2-d84e-6dd84b2c62eb" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a8af6127-0180-983a-8ffd-ec18d569bbb8" }, "outputs": [], "source": [ "all_data = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'],\n", " test.loc[:,'MSSubClass':'SaleCondition']), ignore_index=True)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7ffbc4c8-bf8e-b7ad-8a31-7cd41bfbb1fa" }, "source": [ "## Imputation of missing values ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41f21588-54a2-cccf-ebfb-b82f22818d08" }, "outputs": [], "source": [ "# I have no idea how to do it better. Probably, it is better to do nothing\n", "x = all_data.loc[np.logical_not(all_data[\"LotFrontage\"].isnull()), \"LotArea\"]\n", "y = all_data.loc[np.logical_not(all_data[\"LotFrontage\"].isnull()), \"LotFrontage\"]\n", "# plt.scatter(x, y)\n", "t = (x <= 25000) & (y <= 150)\n", "p = np.polyfit(x[t], y[t], 1)\n", "all_data.loc[all_data['LotFrontage'].isnull(), 'LotFrontage'] = np.polyval(p, all_data.loc[all_data['LotFrontage'].isnull(), 'LotArea'])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "adad1d8c-1f3a-cc8d-eebd-d0f97110a7da" }, "source": [ "There are many features were NaN should be considered as absence of such property. In other cases I replace NaN with most common value" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3bfae950-1716-ccdd-1fe7-c01a0f07cfd3" }, "outputs": [], "source": [ "all_data.loc[all_data.Alley.isnull(), 'Alley'] = 'NoAlley'\n", "all_data.loc[all_data.MasVnrType.isnull(), 'MasVnrType'] = 'None' # no good\n", "all_data.loc[all_data.MasVnrType == 'None', 'MasVnrArea'] = 0\n", "all_data.loc[all_data.BsmtQual.isnull(), 'BsmtQual'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtCond.isnull(), 'BsmtCond'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtExposure.isnull(), 'BsmtExposure'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType1.isnull(), 'BsmtFinType1'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'NoBsmt'\n", "all_data.loc[all_data.BsmtFinType1=='NoBsmt', 'BsmtFinSF1'] = 0\n", "all_data.loc[all_data.BsmtFinType2=='NoBsmt', 'BsmtFinSF2'] = 0\n", "all_data.loc[all_data.BsmtFinSF1.isnull(), 'BsmtFinSF1'] = all_data.BsmtFinSF1.median()\n", "all_data.loc[all_data.BsmtQual=='NoBsmt', 'BsmtUnfSF'] = 0\n", "all_data.loc[all_data.BsmtUnfSF.isnull(), 'BsmtUnfSF'] = all_data.BsmtUnfSF.median()\n", "all_data.loc[all_data.BsmtQual=='NoBsmt', 'TotalBsmtSF'] = 0\n", "all_data.loc[all_data.FireplaceQu.isnull(), 'FireplaceQu'] = 'NoFireplace'\n", "all_data.loc[all_data.GarageType.isnull(), 'GarageType'] = 'NoGarage'\n", "all_data.loc[all_data.GarageFinish.isnull(), 'GarageFinish'] = 'NoGarage'\n", "all_data.loc[all_data.GarageQual.isnull(), 'GarageQual'] = 'NoGarage'\n", "all_data.loc[all_data.GarageCond.isnull(), 'GarageCond'] = 'NoGarage'\n", "all_data.loc[all_data.BsmtFullBath.isnull(), 'BsmtFullBath'] = 0\n", "all_data.loc[all_data.BsmtHalfBath.isnull(), 'BsmtHalfBath'] = 0\n", "all_data.loc[all_data.KitchenQual.isnull(), 'KitchenQual'] = 'TA'\n", "all_data.loc[all_data.MSZoning.isnull(), 'MSZoning'] = 'RL'\n", "all_data.loc[all_data.Utilities.isnull(), 'Utilities'] = 'AllPub'\n", "all_data.loc[all_data.Exterior1st.isnull(), 'Exterior1st'] = 'VinylSd'\n", "all_data.loc[all_data.Exterior2nd.isnull(), 'Exterior2nd'] = 'VinylSd'\n", "all_data.loc[all_data.Functional.isnull(), 'Functional'] = 'Typ'\n", "all_data.loc[all_data.SaleCondition.isnull(), 'SaleCondition'] = 'Normal'\n", "all_data.loc[all_data.SaleCondition.isnull(), 'SaleType'] = 'WD'\n", "all_data.loc[all_data['PoolQC'].isnull(), 'PoolQC'] = 'NoPool'\n", "all_data.loc[all_data['Fence'].isnull(), 'Fence'] = 'NoFence'\n", "all_data.loc[all_data['MiscFeature'].isnull(), 'MiscFeature'] = 'None'\n", "all_data.loc[all_data['Electrical'].isnull(), 'Electrical'] = 'SBrkr'\n", "# only one is null and it has type Detchd\n", "all_data.loc[all_data['GarageArea'].isnull(), 'GarageArea'] = all_data.loc[all_data['GarageType']=='Detchd', 'GarageArea'].mean()\n", "all_data.loc[all_data['GarageCars'].isnull(), 'GarageCars'] = all_data.loc[all_data['GarageType']=='Detchd', 'GarageCars'].median()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9bb00740-bde7-a678-c29f-fb1e2709105b" }, "source": [ "## Normalization ##" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f706c097-313b-d53e-908d-2d02a10868c4" }, "outputs": [], "source": [ "# where we have order we will use numeric\n", "all_data = all_data.replace({'Utilities': {'AllPub': 1, 'NoSeWa': 0, 'NoSewr': 0, 'ELO': 0},\n", " 'Street': {'Pave': 1, 'Grvl': 0 },\n", " 'FireplaceQu': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoFireplace': 0 \n", " },\n", " 'Fence': {'GdPrv': 2, \n", " 'GdWo': 2, \n", " 'MnPrv': 1, \n", " 'MnWw': 1,\n", " 'NoFence': 0},\n", " 'ExterQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1\n", " },\n", " 'ExterCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1\n", " },\n", " 'BsmtQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoBsmt': 0},\n", " 'BsmtExposure': {'Gd': 3, \n", " 'Av': 2, \n", " 'Mn': 1,\n", " 'No': 0,\n", " 'NoBsmt': 0},\n", " 'BsmtCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoBsmt': 0},\n", " 'GarageQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoGarage': 0},\n", " 'GarageCond': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1,\n", " 'NoGarage': 0},\n", " 'KitchenQual': {'Ex': 5, \n", " 'Gd': 4, \n", " 'TA': 3, \n", " 'Fa': 2,\n", " 'Po': 1},\n", " 'Functional': {'Typ': 0,\n", " 'Min1': 1,\n", " 'Min2': 1,\n", " 'Mod': 2,\n", " 'Maj1': 3,\n", " 'Maj2': 4,\n", " 'Sev': 5,\n", " 'Sal': 6} \n", " })" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a22c83af-1b99-80c5-c9e0-ae3b98fc8277" }, "outputs": [], "source": [ "newer_dwelling = all_data.MSSubClass.replace({20: 1, \n", " 30: 0, \n", " 40: 0, \n", " 45: 0,\n", " 50: 0, \n", " 60: 1,\n", " 70: 0,\n", " 75: 0,\n", " 80: 0,\n", " 85: 0,\n", " 90: 0,\n", " 120: 1,\n", " 150: 0,\n", " 160: 0,\n", " 180: 0,\n", " 190: 0})\n", "newer_dwelling.name = 'newer_dwelling'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fcc7f095-ba43-0377-66bc-8ebdfbb18e31" }, "outputs": [], "source": [ "all_data = all_data.replace({'MSSubClass': {20: 'SubClass_20', \n", " 30: 'SubClass_30', \n", " 40: 'SubClass_40', \n", " 45: 'SubClass_45',\n", " 50: 'SubClass_50', \n", " 60: 'SubClass_60',\n", " 70: 'SubClass_70',\n", " 75: 'SubClass_75',\n", " 80: 'SubClass_80',\n", " 85: 'SubClass_85',\n", " 90: 'SubClass_90',\n", " 120: 'SubClass_120',\n", " 150: 'SubClass_150',\n", " 160: 'SubClass_160',\n", " 180: 'SubClass_180',\n", " 190: 'SubClass_190'}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4245cfd6-86f6-4590-fbf3-3f292f3f0495" }, "outputs": [], "source": [ "# The idea is good quality should rise price, poor quality - reduce price\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "qu_list = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "\n", "bad_heating = all_data.HeatingQC.replace({'Ex': 0, \n", " 'Gd': 0, \n", " 'TA': 0, \n", " 'Fa': 1,\n", " 'Po': 1})\n", "bad_heating.name = 'bad_heating'\n", " \n", "MasVnrType_Any = all_data.MasVnrType.replace({'BrkCmn': 1,\n", " 'BrkFace': 1,\n", " 'CBlock': 1,\n", " 'Stone': 1,\n", " 'None': 0})\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace({'Abnorml': 1,\n", " 'Alloca': 1,\n", " 'AdjLand': 1,\n", " 'Family': 1,\n", " 'Normal': 0,\n", " 'Partial': 0})\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "Neighborhood_Good = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Neighborhood_Good'])\n", "Neighborhood_Good[all_data.Neighborhood=='NridgHt'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='Crawfor'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='StoneBr'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='Somerst'] = 1\n", "Neighborhood_Good[all_data.Neighborhood=='NoRidge'] = 1\n", "\n", "# do smth with BsmtFinType1, BsmtFinType2" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8cbb3844-4bef-bbad-3d78-6f834300d110" }, "source": [ "I have no idea what to do with Exterior1st, Exterior2nd, RoofMatl, Condition1, Condition2, BldgType. I'll try convert them into some kind of price brackets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f0fdf976-c5bb-fe0b-3ffe-511197bf11b4" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc], sparse=True)\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b7ecdeae-937b-a58d-befc-4d2b638e1041" }, "outputs": [], "source": [ "p = train.SalePrice/100000\n", "plt.hist(p[pc_pred=='pc1'])\n", "plt.hist(p[pc_pred=='pc2'])\n", "plt.hist(p[pc_pred=='pc3'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b444fe09-fd4b-28e9-a08d-6080f8e9ea1d" }, "outputs": [], "source": [ "price_category = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['pc'])\n", "X_t = pd.get_dummies(all_data.loc[:, columns_for_pc], sparse=True)\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "price_category = price_category.to_sparse()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df571648-bb68-1eed-ad9f-e924911f4dbf" }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "season = all_data.MoSold.replace( {1: 0, \n", " 2: 0, \n", " 3: 0, \n", " 4: 1,\n", " 5: 1, \n", " 6: 1,\n", " 7: 1,\n", " 8: 0,\n", " 9: 0,\n", " 10: 0,\n", " 11: 0,\n", " 12: 0})\n", "season.name = 'season'\n", "\n", "# Numer month is not significant\n", "all_data = all_data.replace({'MoSold': {1: 'Yan', \n", " 2: 'Feb', \n", " 3: 'Mar', \n", " 4: 'Apr',\n", " 5: 'May', \n", " 6: 'Jun',\n", " 7: 'Jul',\n", " 8: 'Avg',\n", " 9: 'Sep',\n", " 10: 'Oct',\n", " 11: 'Nov',\n", " 12: 'Dec'}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7428a0fe-0695-1709-fbee-e57148f7eb39" }, "outputs": [], "source": [ "all_data = all_data.replace({'CentralAir': {'Y': 1, \n", " 'N': 0}})\n", "all_data = all_data.replace({'PavedDrive': {'Y': 1, \n", " 'P': 0,\n", " 'N': 0}})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4a4b1ece-a80f-0071-8020-522e9c75fa26" }, "outputs": [], "source": [ "reconstruct = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Reconstruct'])\n", "reconstruct[all_data.YrSold < all_data.YearRemodAdd] = 1\n", "reconstruct = reconstruct.to_sparse()\n", "\n", "recon_after_buy = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['ReconstructAfterBuy'])\n", "recon_after_buy[all_data.YearRemodAdd >= all_data.YrSold] = 1\n", "recon_after_buy = recon_after_buy.to_sparse()\n", "\n", "build_eq_buy = pd.DataFrame(np.zeros((all_data.shape[0],1)), columns=['Build.eq.Buy'])\n", "build_eq_buy[all_data.YearBuilt >= all_data.YrSold] = 1\n", "build_eq_buy = build_eq_buy.to_sparse()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4aac1977-9a4c-0dfe-8abc-737cb0c945cb" }, "outputs": [], "source": [ "# I hope this will help\n", "all_data.YrSold = 2010 - all_data.YrSold" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "54ee71df-fdcd-00df-6adf-f76453daacdf" }, "outputs": [], "source": [ "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "all_data.GarageYrBlt = all_data.GarageYrBlt.map(year_map)\n", "all_data.loc[all_data['GarageYrBlt'].isnull(), 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "060e746d-16fb-1d41-468a-584b1c5b1cb3" }, "outputs": [], "source": [ "all_data.YearBuilt = all_data.YearBuilt.map(year_map)\n", "all_data.YearRemodAdd = all_data.YearRemodAdd.map(year_map)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6ebb4194-80fb-a91d-e153-cd38688b4a1d" }, "source": [ "Scaling numeric data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2b153f12-8081-ebe4-e503-3c55c8350618" }, "outputs": [], "source": [ "numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n", "\n", "t = all_data[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "all_data[use_max_scater] = all_data[use_max_scater]/all_data[use_max_scater].max()\n", "all_data[use_95_scater] = all_data[use_95_scater]/all_data[use_95_scater].quantile(.95)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "89aace43-ac8c-3c7d-eaf7-c6d85adec796" }, "outputs": [], "source": [ "t = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', \n", " '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', \n", " 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']\n", "\n", "all_data.loc[:, t] = np.log1p(all_data.loc[:, t])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "063dfaf9-7f49-5c62-655b-8c35d5d2f19e" }, "source": [ "## Preparing for sklearn##" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "83da8155-1385-dce5-9e8b-4630ef12ff61" }, "outputs": [], "source": [ "# all classes in sklearn requires numeric data only\n", "# transform categorical variable into binary\n", "X = pd.get_dummies(all_data, sparse=True)\n", "X = X.fillna(0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a64a33b1-9cd0-986a-bc12-76b30a69f912" }, "outputs": [], "source": [ "X = X.drop('RoofMatl_ClyTile', axis=1) # only one is not zero\n", "X = X.drop('Condition2_PosN', axis=1) # only two is not zero\n", "X = X.drop('MSZoning_C (all)', axis=1)\n", "X = X.drop('MSSubClass_SubClass_160', axis=1)\n", "# this features definitely couse overfitting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c3e142c-018d-484d-1b49-997d671a8b12" }, "outputs": [], "source": [ "# add new features\n", "X = pd.concat((X, newer_dwelling, season, reconstruct, recon_after_buy,\n", " qu_list, bad_heating, MasVnrType_Any, price_category, build_eq_buy), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a3d8ee03-439b-1fe1-0170-2948e08f2fb2" }, "source": [ "Next step is guess what new feachers we need to intoduse to make the model better. I'll make a lot of feachers and model wiil choose" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ab19a761-2876-11c8-3fdb-9fbe703872e3" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " # t = [s for s in X.axes[1].get_values() if s not in areas]\n", " t = chain(qu_list.axes[1].get_values(), \n", " ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtCond', 'GarageQual', 'GarageCond',\n", " 'KitchenQual', 'HeatingQC', 'bad_heating', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Reconstruct',\n", " 'ReconstructAfterBuy', 'Build.eq.Buy'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(X), axis=1)\n", "X = pd.concat((X, XP), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cbdd91b4-6ea3-d1bd-3f13-8b92f9265048" }, "outputs": [], "source": [ "X_train = X[:train.shape[0]]\n", "X_test = X[train.shape[0]:]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b4560804-7a54-6f1f-6781-1b173b55e5cd" }, "outputs": [], "source": [ "# the model has become really big\n", "X_train.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b9faea74-013d-0813-9715-44d11b6665ff" }, "outputs": [], "source": [ "y = np.log1p(train.SalePrice)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4be5d5c6-91c1-3f5f-c548-82014a276284" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7b9a32b8-a62d-08c0-523c-d70b8dd23140" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id)\n", "y = y.drop(outliers_id)\n", "# There are difinetly more outliers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fff9d4fd-4ef4-312e-6710-43a7d2a9389b" }, "outputs": [], "source": [ "from sklearn.linear_model import Ridge, RidgeCV, LassoCV, LassoLarsCV\n", "from sklearn.model_selection import cross_val_score\n", "\n", "def rmse_cv(model):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring=\"neg_mean_squared_error\", cv = 5))\n", " return(rmse)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8a3a71df-77eb-e65f-0bc3-180bb92f6022" }, "source": [ "## Model ##" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8a0d1ec7-d9cd-f1b0-5558-d6039636adaf" }, "source": [ "**Ridge**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dcbc8cb0-acc2-2662-514b-fe54ec589bbe" }, "outputs": [], "source": [ "alphas_ridge = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]\n", "cv_rmse_ridge = [rmse_cv(Ridge(alpha = alpha)).mean() \n", " for alpha in alphas_ridge]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b106d046-ad35-c784-62f5-7f963281678d" }, "outputs": [], "source": [ "print (cv_rmse_ridge)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ac9de04a-a9e6-cee7-5e11-45a29850ca52" }, "outputs": [], "source": [ "cv_ridge = pd.Series(cv_rmse_ridge, index = alphas_ridge)\n", "cv_ridge.plot(title = \"Validation Ridge\")\n", "plt.xlabel(\"alphas\")\n", "plt.ylabel(\"rmse\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "07636083-ca2e-e860-ea46-6f0988720791" }, "outputs": [], "source": [ "model_ridge = Ridge(alpha = 10).fit(X_train, y)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e3d7b4ac-189e-2632-d39f-4c2b1654d4cb" }, "outputs": [], "source": [ "cv_ridge.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f31cfd0c-71b3-7067-13db-1e85199f157a" }, "outputs": [], "source": [ "#let's look at the residuals as well:\n", "matplotlib.rcParams['figure.figsize'] = (6.0, 6.0)\n", "\n", "preds_ridge = pd.DataFrame({\"preds Ridge\":model_ridge.predict(X_train), \"true\":y})\n", "preds_ridge[\"residuals\"] = preds_ridge[\"true\"] - preds_ridge[\"preds Ridge\"]\n", "preds_ridge.plot(x = \"preds Ridge\", y = \"residuals\",kind = \"scatter\")" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "5d6ab27c-40c4-c901-29e8-d254db19ff26" }, "source": [ "**Lasso**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b3313ba6-b46d-c4d5-67af-3208966dc564" }, "outputs": [], "source": [ "model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005]).fit(X_train, y)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0e4ba5b2-1f81-38d3-42cb-03942cf6542a" }, "outputs": [], "source": [ "cv_rmse_lasso = rmse_cv(model_lasso).mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a183f839-7605-1869-fefe-1fcd8b5b77fe" }, "outputs": [], "source": [ "print (cv_rmse_lasso)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8b9ba96d-6d1c-d6dc-6a7b-471c5ecabef9" }, "outputs": [], "source": [ "#let's look at the residuals as well:\n", "matplotlib.rcParams['figure.figsize'] = (6.0, 6.0)\n", "\n", "preds_lasso= pd.DataFrame({\"preds Lasso\":model_lasso.predict(X_train), \"true\":y})\n", "preds_lasso[\"residuals\"] = preds_lasso[\"true\"] - preds_lasso[\"preds Lasso\"]\n", "preds_lasso.plot(x = \"preds Lasso\", y = \"residuals\",kind = \"scatter\")" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c72fa4b8-8fd7-44ed-1bb8-b1a4721af464" }, "source": [ "**xgboost model**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "13856ef2-5d71-68b0-9e6c-832ff8b25f48" }, "outputs": [], "source": [ "import xgboost as xgb\n", "dtrain = xgb.DMatrix(X_train, label = y)\n", "dtest = xgb.DMatrix(X_test)\n", "\n", "params = {\"max_depth\":6, \"eta\":0.1}\n", "model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "740ba958-829a-b079-e525-daf19d1c1f52" }, "outputs": [], "source": [ "model.loc[30:,[\"test-rmse-mean\", \"train-rmse-mean\"]].plot()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b446175f-96a6-32fb-3753-7352dbc0c676" }, "outputs": [], "source": [ "model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=6, learning_rate=0.1) #the params were tuned using xgb.cv\n", "model_xgb.fit(X_train, y)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f4de178d-ef5a-6e53-35d8-aa8ce6d40db5" }, "source": [ "**Prediction**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80753de6-b27f-ac21-45cf-5c1405aea617" }, "outputs": [], "source": [ "lasso_preds = np.expm1(model_lasso.predict(X_test))\n", "ridge_preds = np.expm1(model_ridge.predict(X_test))\n", "xgb_preds = np.expm1(model_xgb.predict(X_test))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25c25103-7a20-d89b-1423-be6b74ba251a" }, "outputs": [], "source": [ "preds = 0.65*lasso_preds + 0.15*ridge_preds + 0.2*xgb_preds" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "39a9eeb6-2668-0e4c-d429-6d5237303563" }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\":test.Id, \"SalePrice\":preds})\n", "solution.to_csv(\"lasso_ridge_xgb_cv3.csv\", index = False)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }