{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6ca44af6-af51-968e-a370-070e1fbc6e12" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25372aba-5f6c-3483-4617-af33aadfffcd" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "af6c95f3-7201-64a5-7cc4-7e33ea01f05e" }, "outputs": [], "source": [ "df = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'], test.loc[:, 'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9bcf8a3d-fe87-7cdb-6d94-48d9de01153a" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a43e3b4e-f39c-c9d6-dd00-154919d5f072" }, "outputs": [], "source": [ "print(train.shape)\n", "print(test.shape)\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8a28f9aa-8977-2cc7-0a58-ad34aea8c0f5" }, "outputs": [], "source": [ "df.dtypes.value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8cd58b0f-2a54-a749-9ee1-f571028a0002" }, "outputs": [], "source": [ "object_feats = list(df.dtypes[df.dtypes == \"object\"].index)\n", "print(len(object_feats))\n", "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b9e3884f-b474-1a26-1b69-02a0f2e6a530" }, "outputs": [], "source": [ "int_feats = list(df.dtypes[df.dtypes == \"int64\"].index)\n", "print(len(int_feats))\n", "print(int_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2e0af271-5e51-82a2-4799-bc9233b9e126" }, "outputs": [], "source": [ "float_feats = list(df.dtypes[df.dtypes == \"float64\"].index)\n", "print(len(float_feats))\n", "print(float_feats)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "792c3fe7-8036-4fbd-6904-488f741b1245" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f8d5f56e-fd4f-1308-6e58-e7c971173f3c" }, "outputs": [], "source": [ "na_counts = train.isnull().sum(axis=0)\n", "print('number of features containing NA')\n", "print('train: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = test.isnull().sum(axis=0)\n", "print('test: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = df.isnull().sum(axis=0)\n", "print('all: ', na_counts.loc[na_counts > 0].shape[0])\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fe8ea732-5019-772a-9fa9-149e9d550a68" }, "outputs": [], "source": [ "num_feats = int_feats + float_feats\n", "NA_num = [x for x in list(na_counts.loc[na_counts > 0].index) if x in num_feats]\n", "NA_obj = [x for x in list(na_counts.loc[na_counts > 0].index) if x in object_feats]\n", "print('number of numerical features with NA: ', len(NA_num), '\\nnumber of object features with NA: ', len(NA_obj))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "385b71d0-3795-b5c5-d8ae-74554d7735a1" }, "outputs": [], "source": [ "print(NA_num)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "42298e6d-47ac-436b-3f6c-9a7260186b9c" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "816f9650-0429-43cc-9fa4-3cb9ce773fae" }, "outputs": [], "source": [ "# Let's deal with numerical features first\n", "# We might process some object features in the mean time, if they are about the same properties\n", "# And we use a list to store the \"btw processed\" object features, if their NAs are completely eliminated\n", "NA_obj_btw = []\n", "# Also, we use a list to store the object features that are ordinal categorical\n", "ordianl_obj = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a58c261e-49e1-a256-2db7-c0e29829c566" }, "outputs": [], "source": [ "# LotFrontage\n", "mean_LotFrontage = df.groupby('BldgType').LotFrontage.mean()\n", "for x in list(mean_LotFrontage.index):\n", " df.loc[(df.LotFrontage.isnull()) & (df.BldgType==x), 'LotFrontage'] = mean_LotFrontage[x]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f1c0d100-2142-53c7-fb57-c43d2c5ba9fa" }, "outputs": [], "source": [ "# MasVnrArea\n", "df.loc[df.MasVnrType.isnull(), ['MasVnrType', 'MasVnrArea']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ec7d462b-337e-449c-54e0-5a31919f2928" }, "outputs": [], "source": [ "df.MasVnrType.value_counts(dropna=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fc2ce70f-c575-154d-1e9d-40ee7c7d380a" }, "outputs": [], "source": [ "# We have to fill na correspondingly for both of them\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrType'] = 'None'\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrArea'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "26e6b745-7a43-4de2-b0e1-eb4413212200" }, "outputs": [], "source": [ "# and there is one REAL NaN in MasVnrType, row 1150 shown above\n", "df.loc[df.MasVnrType.isnull(), 'MasVnrType'] = 'BrkFace'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8995df5f-5571-0868-2949-86bf5a8180d0" }, "outputs": [], "source": [ "NA_obj_btw.append('MasVnrType')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "30e80072-b0b5-8055-424c-6f3e6e2f2cec" }, "outputs": [], "source": [ "# features related to basement\n", "base_feats_num = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']\n", "for x in base_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4555717c-8800-02d5-566b-8251f9104850" }, "outputs": [], "source": [ "# maybe it's better to do with all basement features together\n", "# features related to basement\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "91d4b56c-e0ea-9b79-a94c-07b5e67f7f3e" }, "outputs": [], "source": [ "df.loc[df.BsmtFullBath.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "33a8ad08-42a7-d6b6-dfb6-682808ab2014" }, "outputs": [], "source": [ "# it's obvious that these two houses have no basement\n", "df.loc[df.BsmtFullBath.isnull(), base_feats_num] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c65b51a2-6a51-e721-38d0-be517af95bce" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType1.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "166fe80d-778a-e196-6ec5-287d6cb95885" }, "outputs": [], "source": [ "# as in the data description, most NaN means no basement\n", "df.loc[df.BsmtCond.isnull(), base_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "de32be8a-78f6-15af-4c55-dc02d2e126bb" }, "outputs": [], "source": [ "# Maybe there are some REAL NaNs\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d874de32-5da5-5cd7-15b6-b5bfdf29cccd" }, "outputs": [], "source": [ "df.loc[df.BsmtQual.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "04f56ac1-80ad-4b2e-f448-630dd7b7f0b5" }, "outputs": [], "source": [ "df.BsmtQual.mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9aae365e-6a03-f77f-b51b-a2dce27a6408" }, "outputs": [], "source": [ "# yes the BsmtQual measures the height of the basement, and there are two NaNs\n", "df.loc[df.BsmtQual.isnull(), 'BsmtQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9624fe0e-3bbd-421e-5971-ccc8b3e8e521" }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d0d3ef1d-37b0-196d-06d3-f3e6e66edf58" }, "outputs": [], "source": [ "df['BsmtUnfinishRatio'] = df.BsmtUnfSF / df.TotalBsmtSF\n", "grouped = df.groupby('BsmtExposure')\n", "grouped['BsmtUnfinishRatio'].mean().sort_values()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bfee9e94-a1eb-389e-1c31-0e58e0f58219" }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), 'BsmtExposure'] = 'No'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "789ccc32-671f-0190-53a7-ef5ed07e38ef" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d4da5f57-c8a4-4e75-1b38-f1e706c09889" }, "outputs": [], "source": [ "grouped = df.groupby('BsmtFinType2')\n", "grouped['BsmtFinSF2'].mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "624c4cf5-8bfd-bac2-6cfb-34fc7e1f4a99" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'ALQ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fc5187ea-faa3-0948-247e-d99ba18dcd86" }, "outputs": [], "source": [ "# Maybe there are some real NaNs\n", "base_feats = base_feats_num + base_feats_obj\n", "for x in base_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "493d8db3-2c3f-52b8-faa5-840999958058" }, "outputs": [], "source": [ "base_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "90f20490-6e3f-9fb0-a1c3-c68a8a8c4524" }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + ['BsmtQual', 'BsmtCond', 'BsmtFinType1']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2370cee4-fe92-56f1-99a8-ef562a423a1f" }, "outputs": [], "source": [ "# garage\n", "# again, it's better to deal with all garage features in the mean time\n", "garage_feats_num = ['GarageYrBlt', 'GarageCars', 'GarageArea']\n", "garage_feats_obj = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d19906fe-0af1-d012-71be-7bd0a11eefb4" }, "outputs": [], "source": [ "for x in garage_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a4a2d8a8-1435-c595-1c8c-81fe6fb6d730" }, "outputs": [], "source": [ "for x in garage_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "70636982-cb41-e06d-d791-6ef44844f337" }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), garage_feats_num+garage_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df676b39-9282-caf5-f27d-4d47943e83a4" }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), ['GarageCars', 'GarageArea']] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df1fa37e-3d20-448b-0c55-135330cbcdb7" }, "outputs": [], "source": [ "df.loc[df.GarageType.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7aa0bd2d-1eb2-c1c8-5b79-39cef368b662" }, "outputs": [], "source": [ "# as in the data description, NaN means no garage\n", "df.loc[df.GarageType.isnull(), garage_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1827e1ac-6ef7-029a-c37d-b5f239c980a2" }, "outputs": [], "source": [ "df.loc[df.GarageQual.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4b851fca-226d-5a25-7a72-a8b8cb116a82" }, "outputs": [], "source": [ "df.loc[df.GarageCars==1, ['GarageFinish', 'GarageQual', 'GarageCond']].mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ea4590a6-6c2b-6bc4-0870-6da767f035af" }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==1), 'GarageFinish'] = 'Unf'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==1), 'GarageQual'] = 'TA'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==1), 'GarageCond'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "166bffce-56f4-e195-712a-3f249a1a26be" }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==0), 'GarageFinish'] = 'Without'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==0), 'GarageQual'] = 'Without'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==0), 'GarageCond'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f6aebc33-7c30-ded8-3097-d6db5b0b7434" }, "outputs": [], "source": [ "df.loc[df.GarageYrBlt.isnull(), 'GarageYrBlt'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "56712c8f-b9e9-fd9e-2264-dbf343b9d174" }, "outputs": [], "source": [ "garage_feats = garage_feats_num + garage_feats_obj\n", "for x in garage_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "36586c73-8e64-3ab2-8309-181422f5ef7d" }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + garage_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7a047601-517a-8787-48bf-b31d7136c919" }, "outputs": [], "source": [ "# let's check the outcome of the missing values processing\n", "for x in NA_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "96a1893b-6649-8de2-2faf-4de1566a5061" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "39d5294b-2972-4da5-bba8-965f7849d444" }, "outputs": [], "source": [ "# OK, not bad, let's move on to object features\n", "# let's have a look at what have been processed \"BTW\"\n", "NA_obj_btw" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6cb978db-e2a0-bc72-2f4d-4178f52d23a5" }, "outputs": [], "source": [ "NA_obj_new = [x for x in NA_obj if x not in NA_obj_btw]\n", "print(NA_obj_new)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3fc4d404-2f08-d104-afa3-cc1a1c77ec72" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "101ff8bd-2f11-bc4f-1ae5-a4adc52449d3" }, "outputs": [], "source": [ "# MSZoning\n", "print(df.MSZoning.mode())\n", "df.loc[df.MSZoning.isnull(), 'MSZoning'] = 'RL'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5e467835-2af5-402f-f800-823b909b29d4" }, "outputs": [], "source": [ "# Alley\n", "# NaN means no alley\n", "df.loc[df.Alley.isnull(), 'Alley'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "43c1eb4e-cd06-3b81-4720-6e6226aebc8a" }, "outputs": [], "source": [ "# Utilities\n", "print(df.Utilities.mode())\n", "df.loc[df.Utilities.isnull(), 'Utilities'] = 'AllPub'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c9e7dbaf-6a56-d464-a3af-be889e0970e0" }, "outputs": [], "source": [ "# 'Exterior1st' and 'Exterior2nd'\n", "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b139dec5-ea00-0113-2150-346c5017838e" }, "outputs": [], "source": [ "print(df[['Exterior1st', 'Exterior2nd']].mode())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0d3fb029-1a65-d361-8165-f576abe56fb8" }, "outputs": [], "source": [ "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']] = 'VinylSd'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "43ac5572-4fe6-6c20-1573-c88829250898" }, "outputs": [], "source": [ "# Electrical\n", "df.Electrical.mode()\n", "df.loc[df.Electrical.isnull(), 'Electrical'] = 'SBrkr'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "13e89418-74e6-c03c-0b10-2e5c0fc9ee32" }, "outputs": [], "source": [ "# KitchenQual\n", "df.KitchenQual.mode()\n", "df.loc[df.KitchenQual.isnull(), 'KitchenQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "49977fc7-3402-8a66-f705-e227745fd5c7" }, "outputs": [], "source": [ "# Functional\n", "df.Functional.mode()\n", "df.loc[df.Functional.isnull(), 'Functional'] = 'Typ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c840ead5-ac0e-3267-71ae-a4bed10611f8" }, "outputs": [], "source": [ "# FireplaceQu\n", "# NaN means no fireplace\n", "df.loc[df.FireplaceQu.isnull(), 'FireplaceQu'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e78d1883-d713-394e-3a43-be8c6246ead6" }, "outputs": [], "source": [ "# PoolQC\n", "# NaN means no pool\n", "df.loc[df.PoolQC.isnull(), 'PoolQC'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df9e7fe3-dc84-0cc2-abe2-eb64d8d7ffd4" }, "outputs": [], "source": [ "# Fence\n", "# NaN means no fence\n", "df.loc[df.Fence.isnull(), 'Fence'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6516b904-b3df-c0bd-1fa2-44e7a73493c3" }, "outputs": [], "source": [ "# MiscFeature\n", "# NaN means no MiscFeature\n", "df.loc[df.MiscFeature.isnull(), 'MiscFeature'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4b1f636c-631e-cd2c-2d18-9cc6ddd17594" }, "outputs": [], "source": [ "# SaleType\n", "df.SaleType.mode()\n", "df.loc[df.SaleType.isnull(), 'SaleType'] = 'WD'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0e96b8d0-7d82-ea94-947a-81737a815fbb" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3b847c2c-e2ea-0edd-1c94-0a6cb0a9280d" }, "outputs": [], "source": [ "# let's check if there is still NAs\n", "na_counts = df.isnull().sum(axis=0)\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2bb4fd0d-4cd7-cb68-4dc8-43355d50e87d" }, "outputs": [], "source": [ "df.drop(['BsmtUnfinishRatio'], inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "69254cbd-3da0-a051-568c-77c6a343befe" }, "source": [ "" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b7667d26-66e4-76fd-0525-b877586d0d04" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "429a467c-5c3f-f0f1-0f7b-094ce4f7c97c" }, "outputs": [], "source": [ "print('Before', len(object_feats))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b4d2e0fb-4efd-1b71-bec4-23db7bc4a430" }, "outputs": [], "source": [ "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "04cbef63-f3be-7b88-1dc7-35cf4ed128b3" }, "outputs": [], "source": [ "ordinal_words = ['Ex', 'Gd', 'TA', 'Fa', 'Po']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7900f61f-f05e-ec67-724f-5e88ac0b09d3" }, "outputs": [], "source": [ "ordinal_feats = []\n", "for x in object_feats:\n", " flag = 0\n", " for i in ordinal_words:\n", " if df.loc[df[x].str.contains(i, case=True), x].shape[0] > 0:\n", " flag = flag + 1\n", " if flag >= 2:\n", " ordinal_feats.append(x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "96159989-f58a-05f7-74ad-0b1bf8db348a" }, "outputs": [], "source": [ "print(ordinal_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "98945fda-9dc4-16fa-75af-ce8dabf1d04f" }, "outputs": [], "source": [ "for x in ordinal_feats:\n", " print(x, df[x].unique(), len(df[x].unique()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "33ae9f04-0264-92e6-6411-e3b8fc584efb" }, "outputs": [], "source": [ "ordinal_dict = {'Ex':5, 'Gd':4, 'TA':3, 'Fa':2, 'Po':1, 'Without':0}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "06c10a5d-b3a3-63ba-a6d0-b488f3621c49" }, "outputs": [], "source": [ "ordinal_num_feats = []\n", "for x in ordinal_feats:\n", " df[x+'_num'] = df[x].map(ordinal_dict)\n", " ordinal_num_feats.append(x+'_num')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c07d4848-ca18-8b9c-889b-b4e2531ed9ec" }, "outputs": [], "source": [ "df.drop(ordinal_feats, axis=1, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80ee6c98-6f4a-eb0f-34a6-75bd8467493e" }, "outputs": [], "source": [ "ordinal_obj_feats = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'Utilities']\n", "\n", "df['Functional_num'] = df.Functional.replace({'Typ': 6,\n", " 'Min1': 5,\n", " 'Min2': 5,\n", " 'Mod': 4,\n", " 'Maj1': 3,\n", " 'Maj2': 3,\n", " 'Sev': 2,\n", " 'Sal': 1})\n", "df['PavedDrive_num'] = df.PavedDrive.replace({'Y': 3, 'P': 2, 'N': 1})\n", "df['CentralAir_num'] = df.CentralAir.replace({'Y': 1, 'N': 0})\n", "df['Fence_num'] = df.CentralAir.replace({'GdPrv': 2, 'GdWo': 2, 'MnPrv': 1, 'MnWw': 1, 'NoFence': 0})\n", "df['Utilities_num'] = df.Utilities.replace({'AllPub': 1, 'NoSewr': 0, 'NoSeWa': 0, 'ELO': 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0a7f9994-17b3-e812-a37a-441b5154f06e" }, "outputs": [], "source": [ "toDrop_feats = []\n", "toDrop_feats = toDrop_feats + ordinal_obj_feats" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "4a302c39-c042-af74-afc9-7c3c5e1b3d19" }, "source": [ "" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "776c9cdf-d5b9-8413-c7f1-4701e9f3f5e0" }, "source": [ "" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "ffc232b9-4fe7-1dee-1df7-41813f6af4bf" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3ca75d0f-4202-0070-712a-ebb40a053036" }, "outputs": [], "source": [ "toDrop_feats = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80a3c0bd-69a0-ccf2-f371-86da3b1659a3" }, "outputs": [], "source": [ "# some maybe newer\n", "df['newer_dwelling'] = df.MSSubClass.replace({20: 1, 30: 0, 40: 0, 45: 0, 50: 0, 60: 1, 70: 0, 75: 0,\\\n", " 80: 0, 85: 0, 90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})\n", "\n", "# but we still keep the original feature and transform it into categorical\n", "map_MSS = {x: 'Subclass_'+str(x) for x in df.MSSubClass.unique()}\n", "df['MSSubClass'] = df.MSSubClass.replace(map_MSS)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "039f48eb-bc4b-350a-e4f4-0b2deac88730" }, "outputs": [], "source": [ "# it maybe helpful to transform some quality style features into binary\n", "quality_feats = ['OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num',\\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num']\n", "toDrop_feats = toDrop_feats + quality_feats\n", "\n", "all_data = df\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual_num.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual_num.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond_num.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond_num.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond_num.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond_num.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual_num.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual_num.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond_num.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond_num.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual_num.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual_num.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "df_qual = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "df = pd.concat((df, df_qual), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b76cdfe3-d410-b9d7-dc54-4f9b3cab94ae" }, "outputs": [], "source": [ "# for some categorical features, certain levels may imply better quality\n", "toDrop_feats = toDrop_feats + ['MasVnrType', 'SaleCondition', 'Neighborhood']\n", "\n", "map_Mas = {'BrkCmn': 1, 'BrkFace': 1, 'CBlock': 1, 'Stone': 1, 'None': 0}\n", "MasVnrType_Any = all_data.MasVnrType.replace(map_Mas)\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "map_Sale = {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0}\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace(map_Sale)\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "neigh_good_feats = ['NridgHt', 'Crawfor', 'StoneBr', 'Somerst', 'NoRidge']\n", "df['Neighborhood_good'] = 0\n", "df.loc[df.Neighborhood.isin(neigh_good_feats), 'Neighborhood_good'] = 1\n", "\n", "df = pd.concat((df, MasVnrType_Any, SaleCondition_PriceDown), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6627068d-ed29-daca-90cc-6fc942f64760" }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "df['season'] = df.MoSold.replace({1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "18fa92dc-632e-5296-7cd7-3a5ddb0c50e1" }, "outputs": [], "source": [ "# Numer month is not significant, it maybe helpful to transform them into object feature\n", "map_Mo = {1: 'Yan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Avg', 9: 'Sep', 10: 'Oct', \\\n", " 11: 'Nov', 12: 'Dec'}\n", "df = df.replace({'MoSold': map_Mo})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25262501-9d34-c7d2-a5cb-557ff6166382" }, "outputs": [], "source": [ "# some features about years\n", "# sold at the same year as bulit\n", "df['SoldImmediate'] = 0\n", "df.loc[(df.YrSold == df.YearBuilt), 'SoldImmediate'] = 1\n", "\n", "# reconstructed since first built\n", "df['Recon'] = 0\n", "df.loc[(df.YearBuilt < df.YearRemodAdd), 'Recon'] = 1\n", "\n", "# reconstructed after sold\n", "df['ReconAfterSold'] = 0\n", "df.loc[(df.YrSold < df.YearRemodAdd), 'ReconAfterSold'] = 1\n", "\n", "# reconstructed the same year as sold\n", "df['ReconEqualSold'] = 0\n", "df.loc[(df.YrSold == df.YearRemodAdd), 'ReconEqualSold'] = 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c61cc891-b5d3-1dfc-afdf-a1d05b735ec6" }, "outputs": [], "source": [ "# Years are too much, it maybe helpful to devide them into groups, and delete the original ones\n", "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "df.YearBuilt = df.YearBuilt.map(year_map)\n", "df.YearRemodAdd = df.YearRemodAdd.map(year_map)\n", "df.GarageYrBlt = df.GarageYrBlt.map(year_map)\n", "df.loc[df.GarageYrBlt==0, 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "86e85745-30b3-8e92-cf39-e013d07845cf" }, "outputs": [], "source": [ "print(len(toDrop_feats), df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5540e112-fd86-bebe-a839-497d47ec9aaa" }, "outputs": [], "source": [ "t = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'MasVnrType', 'SaleCondition', 'Neighborhood']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f9d0c027-66b0-fd31-29bb-7c7c68e178f8" }, "outputs": [], "source": [ "[x for x in t if x not in toDrop_feats]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4090b7e9-efa1-099a-e6aa-faad68656189" }, "outputs": [], "source": [ "temp = ['MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'Foundation', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating', 'CentralAir', 'Electrical', 'X1stFlrSF', 'X2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', 'X3SsnPorch', 'ScreenPorch', 'PoolArea', 'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType', 'SaleCondition', 'ExterQual_num', 'ExterCond_num', 'BsmtQual_num', 'BsmtCond_num', 'HeatingQC_num', 'KitchenQual_num', 'FireplaceQu_num', 'GarageQual_num', 'GarageCond_num', 'PoolQC_num', 'Functional_num', 'PavedDrive_num', 'CentralAir_num', 'Fence_num', 'Utilities_num', 'newer_dwelling', 'overall_poor_qu', 'overall_good_qu', 'overall_poor_cond', 'overall_good_cond', 'exter_poor_qu', 'exter_good_qu', 'exter_poor_cond', 'exter_good_cond', 'bsmt_poor_cond', 'bsmt_good_cond', 'garage_poor_qu', 'garage_good_qu', 'garage_poor_cond', 'garage_good_cond', 'kitchen_poor_qu', 'kitchen_good_qu', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Neighborhood_good', 'season', 'SoldImmediate', 'Recon', 'ReconAfterSold', 'ReconEqualSold']" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "86b9ea4e-6bc1-82e6-d75e-c7bf96dcfd29" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8bbe6caa-c86f-17c5-0d70-2b264cb9f2d8" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc])\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)\n", "\n", "price_category = pd.DataFrame(np.zeros((df.shape[0],1)), columns=['pc'], index=df.index)\n", "X_t = pd.get_dummies(df.loc[:, columns_for_pc])\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "\n", "toDrop_feats = toDrop_feats + columns_for_pc\n", "df = pd.concat((df, price_category), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "66718e1a-4dd2-8034-b3c3-a9e3ea17d034" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "275110da-921b-e543-7783-6ecfe65da303" }, "outputs": [], "source": [ "numeric_feats = df.dtypes[df.dtypes != \"object\"].index\n", "t = df[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "df[use_max_scater] = df[use_max_scater]/df[use_max_scater].max()\n", "df[use_95_scater] = df[use_95_scater]/df[use_95_scater].quantile(.95)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "55d93c3a-b857-8a7e-44b9-fab9a1756530" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4f1e1ec4-d6cc-5777-fab0-5e969d34c4e7" }, "outputs": [], "source": [ "df_new = df.copy()\n", "from scipy.stats import skew\n", "numeric_feats = list(df_new.dtypes[df_new.dtypes != \"object\"].index)\n", "feat_skewness = df_new[numeric_feats].apply(lambda x: skew(x.dropna()))\n", "skewed_feats = list(feat_skewness[feat_skewness > 0.75].index)\n", "df_new[skewed_feats] = np.log1p(df_new[skewed_feats])\n", "\n", "df = df_new.copy()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6270e0a6-0faf-a961-ee52-2670f41cda4c" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "03d7ea59-d137-18ca-a7d4-84388d813729" }, "outputs": [], "source": [ "" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b2707271-c8d7-8310-cb33-a193afc2f20d" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "24409f93-6d2e-e412-1d62-9cfacecea250" }, "outputs": [], "source": [ "df_new = pd.get_dummies(df)\n", "print(df.shape, df_new.shape)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f9a89a7d-792b-4e93-260e-fd42e5a96673" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bdbc9058-0e4f-8f87-2cf4-e4791ba692e6" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " t = chain(df_qual.axes[1].get_values(), \n", " ['OverallQual_num', 'OverallCond_num', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', \\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'HeatingQC_num', \\\n", " 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Recon',\n", " 'ReconAfterSold', 'SoldImmediate'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(df_new), axis=1)\n", "df_new = pd.concat((df_new, XP), axis=1)\n", "\n", "df_new.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "67b9da3c-c3e8-4d0e-34e6-7b9dadf85f42" }, "outputs": [], "source": [ "df_final = df_new.copy()\n", "\n", "X_train = df_final[:train.shape[0]]\n", "print(X_train.shape)\n", "y_train = np.log1p(train.SalePrice)\n", "print(y_train.shape)\n", "X_test = df_final[test.shape[0]+1:]\n", "print(X_test.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "390535c1-457a-cd7b-17ac-9d11babebd58" }, "outputs": [], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "from sklearn.cross_validation import cross_val_score\n", "from sklearn.metrics import make_scorer, mean_squared_error\n", "\n", "def rmse_cv(model, X_train, y_train):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring=\"neg_mean_squared_error\", cv=10))\n", " return(rmse)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b36ac83c-992c-824b-1a8c-b52a130dd067" }, "outputs": [], "source": [ "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result = pd.Series(cv_lasso, index = alphas)\n", "result.plot()\n", "result.min()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "25779747-050c-822c-9c66-a952714a3a3e" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f70e0c21-4b43-4100-e8db-e41d38c7412c" }, "outputs": [], "source": [ "lasso_test = Lasso(alpha=0.00005, max_iter=1000).fit(X_train, y_train)\n", "y_test = lasso_test.predict(X_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b4329084-cd99-fad5-4c7c-af4358e2b4f3" }, "outputs": [], "source": [ "dfy = pd.DataFrame({\"id\": train.Id, \"y_real\": y_train, \"y_test\": y_test, \"residual\": y_test - y_train},\\\n", " columns=['id', 'y_real', 'y_test', 'residual'])\n", "\n", "outliers_id = np.array([524, 1299])\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "738c221d-806a-6bf1-2370-673400adf8b3" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='y_test', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 2], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b5c574b3-f850-4abf-96e9-86dc907d9d25" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='residual', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 3], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "da518f47-ec2a-90a3-2a06-c84b1cb5d464" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "ax = plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6e387926-bea3-68d4-4708-d901a98fd52f" }, "source": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "749dc43a-fd64-29f7-3271-1eb8b1d3d131" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id, axis=0)\n", "y_train = y_train.drop(outliers_id, axis=0)\n", "\n", "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result = pd.Series(cv_lasso, index = alphas)\n", "result.plot()\n", "result.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9510e40d-e71c-7e16-56d7-22427f9edda3" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "793ebe18-e475-b314-76ec-a9c223a8602d" }, "outputs": [], "source": [ "lasso = Lasso(alpha=0.00075, max_iter=1000).fit(X_train, y_train)\n", "y_pred = lasso.predict(X_test)\n", "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_pred)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"lasso_sol.csv\", index = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "457cb2f8-6410-7d3e-25f6-50de2ecfb83f" }, "outputs": [], "source": [ "import xgboost as xgb\n", "regr = xgb.XGBRegressor(\n", " colsample_bytree=0.2,\n", " gamma=0.0,\n", " learning_rate=0.01,\n", " max_depth=4,\n", " min_child_weight=1.5,\n", " n_estimators=7200, \n", " reg_alpha=0.9,\n", " reg_lambda=0.6,\n", " subsample=0.2,\n", " seed=42,\n", " silent=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f9e43d80-2ce3-6f22-5639-dce6655d3f25" }, "outputs": [], "source": [ "from sklearn.metrics import mean_squared_error\n", "def rmse(y_true, y_pred):\n", " return np.sqrt(mean_squared_error(y_true, y_pred))\n", "\n", "#regr.fit(X_train, y_train)\n", "#y_pred = regr.predict(X_train)\n", "print(\"XGBoost score on training set: \", rmse_cv(regr,X_train, y_train))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7ce830f5-1303-5ac9-8703-70cc1130630e" }, "outputs": [], "source": [ "y_sol = regr.predict(X_test)\n", "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_sol)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"axg_sol.csv\", index = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c09c0ecd-6b19-dcba-39e9-72e7135e3c98" }, "outputs": [], "source": [ "from sklearn import ensemble, tree, linear_model\n", "from sklearn.model_selection import train_test_split, cross_val_score\n", "from sklearn.metrics import r2_score, mean_squared_error\n", "from sklearn.utils import shuffle\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "21c4e5ba-ece4-da50-b5f3-0cd3bf17dea2" }, "outputs": [], "source": [ "GBest = ensemble.GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=3, max_features='sqrt',\n", " min_samples_leaf=15, min_samples_split=10, loss='huber').fit(X_train, y_train)\n", "y_pred = GBest.predict(X_train)\n", "print(\"Gradient boosting score on training set: \", rmse(y_train, y_pred))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "20f7eada-06e7-0ee8-6b93-88b68ae2cdb1" }, "outputs": [], "source": [ "y_sol = GBest.predict(X_test)\n", "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_sol)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"GBest_sol.csv\", index = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c1f520e-b5f3-e328-4967-56ddfe88ea4a" }, "outputs": [], "source": [ "" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }