{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "134fbade-9ad1-11e3-fa05-1feef6cd4726" }, "source": [ "heyheyhey" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ca3bbf74-8725-7ff8-8b2e-391103c6080c" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "16c8328b-42e2-ea5e-19a9-6f4f89f1dd8e" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "68909ba3-0c40-d51c-0ef0-517f72db890f" }, "outputs": [], "source": [ "df = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'], test.loc[:, 'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "49ccb83a-d5bd-726f-c98b-08016a5c1e28" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Data Summary>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fdbeb8d5-6574-7742-5a9f-1b76e2968d93" }, "outputs": [], "source": [ "print(train.shape)\n", "print(test.shape)\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c3bc6fb2-d55e-a79a-7260-047ecb1c061e" }, "outputs": [], "source": [ "df.dtypes.value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "62b94f85-0102-7475-db21-57bf2b4a71db" }, "outputs": [], "source": [ "object_feats = list(df.dtypes[df.dtypes == \"object\"].index)\n", "print(len(object_feats))\n", "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c4336de5-7d4f-db9b-5c3a-b50534c2f1e8" }, "outputs": [], "source": [ "int_feats = list(df.dtypes[df.dtypes == \"int64\"].index)\n", "print(len(int_feats))\n", "print(int_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2473124d-55d8-553a-9300-3f22720d77dd" }, "outputs": [], "source": [ "float_feats = list(df.dtypes[df.dtypes == \"float64\"].index)\n", "print(len(float_feats))\n", "print(float_feats)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "498b5b11-1a15-e1f0-1899-923c9e7a1521" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>Missing value processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "79ca6b24-cb08-6d19-cf6b-7df274623f28" }, "outputs": [], "source": [ "na_counts = train.isnull().sum(axis=0)\n", "print('number of features containing NA')\n", "print('train: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = test.isnull().sum(axis=0)\n", "print('test: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = df.isnull().sum(axis=0)\n", "print('all: ', na_counts.loc[na_counts > 0].shape[0])\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d776062d-201d-59b5-5564-b53ad74970dc" }, "outputs": [], "source": [ "num_feats = int_feats + float_feats\n", "NA_num = [x for x in list(na_counts.loc[na_counts > 0].index) if x in num_feats]\n", "NA_obj = [x for x in list(na_counts.loc[na_counts > 0].index) if x in object_feats]\n", "print('number of numerical features with NA: ', len(NA_num), '\\nnumber of object features with NA: ', len(NA_obj))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "17c29ccd-0acc-91b5-9058-a35d94138973" }, "outputs": [], "source": [ "print(NA_num)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e73052e7-2a2a-09fb-cafd-f1cc725b2c71" }, "source": [ "### 1. Numerical features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a1a2f49a-310f-ec8b-b7be-444817c27420", "collapsed": true }, "outputs": [], "source": [ "# Let's deal with numerical features first\n", "# We might process some object features in the mean time, if they are about the same properties\n", "# And we use a list to store the \"btw processed\" object features, if their NAs are completely eliminated\n", "NA_obj_btw = []\n", "# Also, we use a list to store the object features that are ordinal categorical\n", "ordianl_obj = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a6a2a47b-ac2c-bf47-d8c6-0c3d00a40fed" }, "outputs": [], "source": [ "# LotFrontage\n", "mean_LotFrontage = df.groupby('BldgType').LotFrontage.mean()\n", "for x in list(mean_LotFrontage.index):\n", " df.loc[(df.LotFrontage.isnull()) & (df.BldgType==x), 'LotFrontage'] = mean_LotFrontage[x]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b9883cd3-1bfb-8fdf-f2cc-324ea63319f1" }, "outputs": [], "source": [ "# MasVnrArea\n", "df.loc[df.MasVnrType.isnull(), ['MasVnrType', 'MasVnrArea']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8583224b-b644-2875-0a46-5608a4471aa0" }, "outputs": [], "source": [ "df.MasVnrType.value_counts(dropna=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ed251f03-0751-4b32-e7fb-e00e7375ffe2" }, "outputs": [], "source": [ "# We have to fill na correspondingly for both of them\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrType'] = 'None'\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrArea'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d2952aec-48ca-7fe6-7cc4-9a42faae6112", "collapsed": true }, "outputs": [], "source": [ "# and there is one REAL NaN in MasVnrType, row 1150 shown above\n", "df.loc[df.MasVnrType.isnull(), 'MasVnrType'] = 'BrkFace'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9bd25e3a-4b11-0405-beed-668fdc41bbf4", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw.append('MasVnrType')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9363ce3d-8f00-903e-5226-116a45ab9352" }, "outputs": [], "source": [ "# features related to basement\n", "base_feats_num = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']\n", "for x in base_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fc40bfe4-1d75-525b-c77a-f7456b3210c0" }, "outputs": [], "source": [ "# maybe it's better to do with all basement features together\n", "# features related to basement\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dac71101-0fac-4489-6fea-7ed6a9854ab9" }, "outputs": [], "source": [ "df.loc[df.BsmtFullBath.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "49043a51-641a-44ef-f445-cec267280d3d", "collapsed": true }, "outputs": [], "source": [ "# it's obvious that these two houses have no basement\n", "df.loc[df.BsmtFullBath.isnull(), base_feats_num] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "27c42b79-c3c8-04c9-951d-e5fdce8dd74c" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType1.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5df8f185-7f63-0949-3210-d23cefead834", "collapsed": true }, "outputs": [], "source": [ "# as in the data description, most NaN means no basement\n", "df.loc[df.BsmtCond.isnull(), base_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4662fc69-59ab-f2db-2bf1-23f14ea60923" }, "outputs": [], "source": [ "# Maybe there are some REAL NaNs\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5d298f49-1254-f55a-d0a8-ed4461d90a50" }, "outputs": [], "source": [ "df.loc[df.BsmtQual.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bade861d-d253-5033-691e-8f563d5c92ab" }, "outputs": [], "source": [ "df.BsmtQual.mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "252e8a4e-20e3-5c98-30c2-8002e545e549", "collapsed": true }, "outputs": [], "source": [ "# yes the BsmtQual measures the height of the basement, and there are two NaNs\n", "df.loc[df.BsmtQual.isnull(), 'BsmtQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c138290b-a0b2-2aa7-acd1-007cd2f88ee4" }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "32484191-de79-02d6-c777-68839f578fb3" }, "outputs": [], "source": [ "df['BsmtUnfinishRatio'] = df.BsmtUnfSF / df.TotalBsmtSF\n", "grouped = df.groupby('BsmtExposure')\n", "grouped['BsmtUnfinishRatio'].mean().sort_values()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5329c24b-60d3-80c4-389b-57f62e73c66f", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), 'BsmtExposure'] = 'No'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9235f795-873c-e430-d0ae-8303e51bea80" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "20111e33-7b2a-5cf4-2932-e47659003452" }, "outputs": [], "source": [ "grouped = df.groupby('BsmtFinType2')\n", "grouped['BsmtFinSF2'].mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "580ff86e-43a4-2a17-7e66-16e8be359d25", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'ALQ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b36c3f07-1cfc-e11f-41eb-198ee7d3437d" }, "outputs": [], "source": [ "# Maybe there are some real NaNs\n", "base_feats = base_feats_num + base_feats_obj\n", "for x in base_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d05793b6-9043-8ff3-8582-c01c9bc8d03c" }, "outputs": [], "source": [ "base_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2f1c2f0a-be22-18b1-7960-4f229aa86748", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + ['BsmtQual', 'BsmtCond', 'BsmtFinType1']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "adde0ba6-579b-96e1-7181-13dac860a5a5", "collapsed": true }, "outputs": [], "source": [ "# garage\n", "# again, it's better to deal with all garage features in the mean time\n", "garage_feats_num = ['GarageYrBlt', 'GarageCars', 'GarageArea']\n", "garage_feats_obj = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7dd6d495-c346-2d8a-8e40-f1cf3acb6982" }, "outputs": [], "source": [ "for x in garage_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "431502f7-d1e3-b6e5-7670-6191e62931a7" }, "outputs": [], "source": [ "for x in garage_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "078f6324-942e-171a-621d-09bd0d588cc1" }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), garage_feats_num+garage_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ff8748e0-a3c0-0b8d-5a5b-f7dcb4376ffb", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), ['GarageCars', 'GarageArea']] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4fae0e85-d2ee-779c-eafd-ed1c19a922a7" }, "outputs": [], "source": [ "df.loc[df.GarageType.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b62627cd-4700-6fc6-ac14-d2968bf43e31", "collapsed": true }, "outputs": [], "source": [ "# as in the data description, NaN means no garage\n", "df.loc[df.GarageType.isnull(), garage_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3e12d9b9-b361-6c25-dd57-f9e303381671" }, "outputs": [], "source": [ "df.loc[df.GarageQual.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "58546658-b17b-e9c3-34e8-138c14d01f7a" }, "outputs": [], "source": [ "df.loc[df.GarageCars==1, ['GarageFinish', 'GarageQual', 'GarageCond']].mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7f815ed1-939a-3e1e-d52b-7c9d6931d730", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==1), 'GarageFinish'] = 'Unf'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==1), 'GarageQual'] = 'TA'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==1), 'GarageCond'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "656deaa7-a9f6-20d3-7a98-93a457d969bd", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==0), 'GarageFinish'] = 'Without'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==0), 'GarageQual'] = 'Without'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==0), 'GarageCond'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8a12aa42-1f7f-acc6-57c7-e061ea2ea6e0", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageYrBlt.isnull(), 'GarageYrBlt'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "495daf77-5f66-d429-3195-a7b9dc1e936b" }, "outputs": [], "source": [ "garage_feats = garage_feats_num + garage_feats_obj\n", "for x in garage_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c50434fe-517d-3814-0ae0-0e599dbe4c40", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + garage_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eb6eb20f-5960-fa29-79e8-abc407e8819a" }, "outputs": [], "source": [ "# let's check the outcome of the missing values processing\n", "for x in NA_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "4e0a0f50-338e-cc8f-8fc1-d439e9d30726" }, "source": [ "### 2. Object features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41ccea2b-1730-d23d-312b-9e8f14813d17" }, "outputs": [], "source": [ "# OK, not bad, let's move on to object features\n", "# let's have a look at what have been processed \"BTW\"\n", "NA_obj_btw" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1826fe36-2fc4-0987-1b05-f62b82d52236" }, "outputs": [], "source": [ "NA_obj_new = [x for x in NA_obj if x not in NA_obj_btw]\n", "print(NA_obj_new)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e0b911a9-425e-8fe7-b809-d0a40005156c" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5a6a8f39-790e-c003-6312-4be01383f576" }, "outputs": [], "source": [ "# MSZoning\n", "print(df.MSZoning.mode())\n", "df.loc[df.MSZoning.isnull(), 'MSZoning'] = 'RL'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "58e91478-4a43-f294-0292-8a435cebfb4b", "collapsed": true }, "outputs": [], "source": [ "# Alley\n", "# NaN means no alley\n", "df.loc[df.Alley.isnull(), 'Alley'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "be347978-045e-159f-acc8-cc8b9e10b8fc" }, "outputs": [], "source": [ "# Utilities\n", "print(df.Utilities.mode())\n", "df.loc[df.Utilities.isnull(), 'Utilities'] = 'AllPub'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e0a9159c-cf60-befa-a91f-0bf6515ea394" }, "outputs": [], "source": [ "# 'Exterior1st' and 'Exterior2nd'\n", "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a998bc66-b750-b843-bf81-b7f26b709f96" }, "outputs": [], "source": [ "print(df[['Exterior1st', 'Exterior2nd']].mode())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8977ff8d-afd0-47ad-9061-bc460f5c3bf2", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']] = 'VinylSd'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "981c55d5-8cc0-b203-5b5c-22ebf10919de", "collapsed": true }, "outputs": [], "source": [ "# Electrical\n", "df.Electrical.mode()\n", "df.loc[df.Electrical.isnull(), 'Electrical'] = 'SBrkr'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "319872e9-d028-6943-4577-02faadfc202f", "collapsed": true }, "outputs": [], "source": [ "# KitchenQual\n", "df.KitchenQual.mode()\n", "df.loc[df.KitchenQual.isnull(), 'KitchenQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a58f2f38-84b5-3a00-07a6-bd08051d77f7", "collapsed": true }, "outputs": [], "source": [ "# Functional\n", "df.Functional.mode()\n", "df.loc[df.Functional.isnull(), 'Functional'] = 'Typ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6cef0123-7057-532e-f21d-b45c6f3e937d", "collapsed": true }, "outputs": [], "source": [ "# FireplaceQu\n", "# NaN means no fireplace\n", "df.loc[df.FireplaceQu.isnull(), 'FireplaceQu'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "25a5c231-5d51-fc68-33a1-13f24e81115e", "collapsed": true }, "outputs": [], "source": [ "# PoolQC\n", "# NaN means no pool\n", "df.loc[df.PoolQC.isnull(), 'PoolQC'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b98e9174-08f8-1769-543d-893cd45845d3", "collapsed": true }, "outputs": [], "source": [ "# Fence\n", "# NaN means no fence\n", "df.loc[df.Fence.isnull(), 'Fence'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fc94cf8c-6512-26cb-d4eb-605461eb6b39", "collapsed": true }, "outputs": [], "source": [ "# MiscFeature\n", "# NaN means no MiscFeature\n", "df.loc[df.MiscFeature.isnull(), 'MiscFeature'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c8debfcd-a705-f34d-2d56-28236365f4f6", "collapsed": true }, "outputs": [], "source": [ "# SaleType\n", "df.SaleType.mode()\n", "df.loc[df.SaleType.isnull(), 'SaleType'] = 'WD'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "30a83003-e0f3-a10a-2b04-ba0038681a9b" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ef556ee6-809c-b7bb-8231-db73d62e6d79" }, "outputs": [], "source": [ "# let's check if there is still NAs\n", "na_counts = df.isnull().sum(axis=0)\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "99cbfefd-b45f-f9eb-f787-2c5099c2a5ce", "collapsed": true }, "outputs": [], "source": [ "df.drop(['BsmtUnfinishRatio'], inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6a30bfcd-bbdd-f70b-b349-0d7fbdaaa227" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>End of Missing Value Processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a1937517-7a63-b120-4af6-a2df8c888533" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>Transform ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1a29cc90-9fa6-7218-c8dd-6001a3bfe710" }, "outputs": [], "source": [ "print('Before', len(object_feats))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b3aa0c71-8932-4aa6-a948-8a34c87f4e11" }, "outputs": [], "source": [ "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9c4d5361-f2e3-e657-183a-f02592640a5b", "collapsed": true }, "outputs": [], "source": [ "ordinal_words = ['Ex', 'Gd', 'TA', 'Fa', 'Po']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a40ac473-b7c8-61da-5e2e-a6206a85ca52", "collapsed": true }, "outputs": [], "source": [ "ordinal_feats = []\n", "for x in object_feats:\n", " flag = 0\n", " for i in ordinal_words:\n", " if df.loc[df[x].str.contains(i, case=True), x].shape[0] > 0:\n", " flag = flag + 1\n", " if flag >= 2:\n", " ordinal_feats.append(x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0b954aa6-bd1d-2977-8670-9c9037aef404" }, "outputs": [], "source": [ "print(ordinal_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "365f4f88-dbf6-238a-866c-9ae359df832d" }, "outputs": [], "source": [ "for x in ordinal_feats:\n", " print(x, df[x].unique(), len(df[x].unique()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "08985333-f265-1879-0ebb-06ca8e7ef0f9", "collapsed": true }, "outputs": [], "source": [ "ordinal_dict = {'Ex':5, 'Gd':4, 'TA':3, 'Fa':2, 'Po':1, 'Without':0}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "86b399c8-1a48-e10a-1743-2dc715ecd7ca", "collapsed": true }, "outputs": [], "source": [ "ordinal_num_feats = []\n", "for x in ordinal_feats:\n", " df[x+'_num'] = df[x].map(ordinal_dict)\n", " ordinal_num_feats.append(x+'_num')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "420e6178-f5e2-2d83-eaf6-4ae86b9022c5", "collapsed": true }, "outputs": [], "source": [ "df.drop(ordinal_feats, axis=1, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c94fc284-ec6e-b1ed-1f0c-12c21971de4d", "collapsed": true }, "outputs": [], "source": [ "ordinal_obj_feats = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'Utilities']\n", "\n", "df['Functional_num'] = df.Functional.replace({'Typ': 6,\n", " 'Min1': 5,\n", " 'Min2': 5,\n", " 'Mod': 4,\n", " 'Maj1': 3,\n", " 'Maj2': 3,\n", " 'Sev': 2,\n", " 'Sal': 1})\n", "df['PavedDrive_num'] = df.PavedDrive.replace({'Y': 3, 'P': 2, 'N': 1})\n", "df['CentralAir_num'] = df.CentralAir.replace({'Y': 1, 'N': 0})\n", "df['Fence_num'] = df.CentralAir.replace({'GdPrv': 2, 'GdWo': 2, 'MnPrv': 1, 'MnWw': 1, 'NoFence': 0})\n", "df['Utilities_num'] = df.Utilities.replace({'AllPub': 1, 'NoSewr': 0, 'NoSeWa': 0, 'ELO': 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8c5cfa7d-9828-2618-360e-d252feca47a4", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []\n", "toDrop_feats = toDrop_feats + ordinal_obj_feats" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "74b4ec6e-9a8d-160a-d99a-9b473a594095" }, "outputs": [], "source": [ "df.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "bb92408d-dda3-2a2b-2d96-332ed947a498" }, "source": [ "**>>>>>>>>>>>>>>>>End of transformation of ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8b9c5a85-dc9a-f9c0-a928-10eef72f9898" }, "source": [ "**>>>>>>>>>>>>>>>>Some arguable feature engineering>>>>>>>>>>>>>>>>>>**\n", "\n", "**For any cells in this part, if you don't want it, just mark or delete the whole well, it won't affect cells after.**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d3ecc3e3-8fb3-158e-6375-ec2d54b1e4f5" }, "source": [ "### 1. Try to make quality features and some categorical features more \"significant\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ecc3ce6b-80a2-06be-c22a-ba56268116e2", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3fd2673c-119b-25df-b939-349a6d6e2022", "collapsed": true }, "outputs": [], "source": [ "# some maybe newer\n", "df['newer_dwelling'] = df.MSSubClass.replace({20: 1, 30: 0, 40: 0, 45: 0, 50: 0, 60: 1, 70: 0, 75: 0,\\\n", " 80: 0, 85: 0, 90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})\n", "\n", "# but we still keep the original feature and transform it into categorical\n", "map_MSS = {x: 'Subclass_'+str(x) for x in df.MSSubClass.unique()}\n", "df['MSSubClass'] = df.MSSubClass.replace(map_MSS)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "59fc452c-109e-d5ad-9010-4034b8ae4976" }, "outputs": [], "source": [ "# it maybe helpful to transform some quality style features into binary\n", "quality_feats = ['OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num',\\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num']\n", "toDrop_feats = toDrop_feats + quality_feats\n", "\n", "all_data = df\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual_num.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual_num.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond_num.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond_num.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond_num.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond_num.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual_num.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual_num.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond_num.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond_num.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual_num.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual_num.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "df_qual = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "df = pd.concat((df, df_qual), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4a7a13cc-73c4-ce81-b5b6-917bd8a0fe5a", "collapsed": true }, "outputs": [], "source": [ "# for some categorical features, certain levels may imply better quality\n", "toDrop_feats = toDrop_feats + ['MasVnrType', 'SaleCondition', 'Neighborhood']\n", "\n", "map_Mas = {'BrkCmn': 1, 'BrkFace': 1, 'CBlock': 1, 'Stone': 1, 'None': 0}\n", "MasVnrType_Any = all_data.MasVnrType.replace(map_Mas)\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "map_Sale = {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0}\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace(map_Sale)\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "neigh_good_feats = ['NridgHt', 'Crawfor', 'StoneBr', 'Somerst', 'NoRidge']\n", "df['Neighborhood_good'] = 0\n", "df.loc[df.Neighborhood.isin(neigh_good_feats), 'Neighborhood_good'] = 1\n", "\n", "df = pd.concat((df, MasVnrType_Any, SaleCondition_PriceDown), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d9d84954-58cf-d99d-5517-53ffb698256c", "collapsed": true }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "df['season'] = df.MoSold.replace({1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8579de73-46f5-ed17-99f4-a4bd6e73e070" }, "outputs": [], "source": [ "# Numer month is not significant, it maybe helpful to transform them into object feature\n", "map_Mo = {1: 'Yan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Avg', 9: 'Sep', 10: 'Oct', \\\n", " 11: 'Nov', 12: 'Dec'}\n", "df = df.replace({'MoSold': map_Mo})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "70295529-85e0-232b-3825-04781182e363", "collapsed": true }, "outputs": [], "source": [ "# some features about years\n", "# sold at the same year as bulit\n", "df['SoldImmediate'] = 0\n", "df.loc[(df.YrSold == df.YearBuilt), 'SoldImmediate'] = 1\n", "\n", "# reconstructed since first built\n", "df['Recon'] = 0\n", "df.loc[(df.YearBuilt < df.YearRemodAdd), 'Recon'] = 1\n", "\n", "# reconstructed after sold\n", "df['ReconAfterSold'] = 0\n", "df.loc[(df.YrSold < df.YearRemodAdd), 'ReconAfterSold'] = 1\n", "\n", "# reconstructed the same year as sold\n", "df['ReconEqualSold'] = 0\n", "df.loc[(df.YrSold == df.YearRemodAdd), 'ReconEqualSold'] = 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "00ec66bb-52e3-13ee-ba88-e92bed5c5001", "collapsed": true }, "outputs": [], "source": [ "# Years are too much, it maybe helpful to devide them into groups, and delete the original ones\n", "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "df.YearBuilt = df.YearBuilt.map(year_map)\n", "df.YearRemodAdd = df.YearRemodAdd.map(year_map)\n", "df.GarageYrBlt = df.GarageYrBlt.map(year_map)\n", "df.loc[df.GarageYrBlt==0, 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9150878b-af1c-7580-33a9-2aad664246dc" }, "outputs": [], "source": [ "print(len(toDrop_feats), df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e21e16a2-0bfe-57fc-83d5-f703b9fb7956", "collapsed": true }, "outputs": [], "source": [ "t = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'MasVnrType', 'SaleCondition', 'Neighborhood']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "09e2c855-fb9a-890d-bff3-dc3f878c29bb" }, "outputs": [], "source": [ "[x for x in t if x not in toDrop_feats]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9632a038-1796-eaba-f358-7e79d78c7e2d", "collapsed": true }, "outputs": [], "source": [ "temp = ['MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'Foundation', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating', 'CentralAir', 'Electrical', 'X1stFlrSF', 'X2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', 'X3SsnPorch', 'ScreenPorch', 'PoolArea', 'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType', 'SaleCondition', 'ExterQual_num', 'ExterCond_num', 'BsmtQual_num', 'BsmtCond_num', 'HeatingQC_num', 'KitchenQual_num', 'FireplaceQu_num', 'GarageQual_num', 'GarageCond_num', 'PoolQC_num', 'Functional_num', 'PavedDrive_num', 'CentralAir_num', 'Fence_num', 'Utilities_num', 'newer_dwelling', 'overall_poor_qu', 'overall_good_qu', 'overall_poor_cond', 'overall_good_cond', 'exter_poor_qu', 'exter_good_qu', 'exter_poor_cond', 'exter_good_cond', 'bsmt_poor_cond', 'bsmt_good_cond', 'garage_poor_qu', 'garage_good_qu', 'garage_poor_cond', 'garage_good_cond', 'kitchen_poor_qu', 'kitchen_good_qu', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Neighborhood_good', 'season', 'SoldImmediate', 'Recon', 'ReconAfterSold', 'ReconEqualSold']" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "94e6ee29-ce5a-b0f0-d617-a81070c2c1ac" }, "source": [ "### 2. Do SVC for some features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c909fba1-f235-8cff-f99a-8a24e1591d3d" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc])\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)\n", "\n", "price_category = pd.DataFrame(np.zeros((df.shape[0],1)), columns=['pc'], index=df.index)\n", "X_t = pd.get_dummies(df.loc[:, columns_for_pc])\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "\n", "toDrop_feats = toDrop_feats + columns_for_pc\n", "df = pd.concat((df, price_category), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "144d427a-32fa-ae5e-a0b6-f3fa1dbc6a98" }, "source": [ "### 3. Scale the numerical data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4110fbe8-04f8-a531-de13-a9f1e291506f", "collapsed": true }, "outputs": [], "source": [ "numeric_feats = df.dtypes[df.dtypes != \"object\"].index\n", "t = df[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "df[use_max_scater] = df[use_max_scater]/df[use_max_scater].max()\n", "df[use_95_scater] = df[use_95_scater]/df[use_95_scater].quantile(.95)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f5d54ac2-2429-71e2-3a65-39aa894b7c75" }, "source": [ "### 4. Log skewed data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ce3dac86-1b6c-8ad2-48c3-ae052556f3a7", "collapsed": true }, "outputs": [], "source": [ "df_new = df.copy()\n", "from scipy.stats import skew\n", "numeric_feats = list(df_new.dtypes[df_new.dtypes != \"object\"].index)\n", "feat_skewness = df_new[numeric_feats].apply(lambda x: skew(x.dropna()))\n", "skewed_feats = list(feat_skewness[feat_skewness > 0.75].index)\n", "df_new[skewed_feats] = np.log1p(df_new[skewed_feats])\n", "\n", "df = df_new.copy()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "497df3ae-43c0-785c-b4cb-a1b186d19cd2" }, "source": [ "### 5. Drop some features that have been used to create new features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f8cca438-ac51-7ba8-51f8-4ab0d703903a" }, "outputs": [], "source": [ "# print(len(toDrop_feats), toDrop_feats)\n", "# df.drop(toDrop_feats, inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8d84540e-38bb-e50f-1ba3-8e37ea395d36" }, "source": [ "### 6. Get dummy features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ab5308e3-05ad-b409-75ca-25853305ef52" }, "outputs": [], "source": [ "df_new = pd.get_dummies(df)\n", "print(df.shape, df_new.shape)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "68f77f90-bb1e-9341-b575-f41c508626e8" }, "source": [ "### 7. Create interact features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cf7db8a7-b89e-78b0-7fec-0f3af7eefac5" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " t = chain(df_qual.axes[1].get_values(), \n", " ['OverallQual_num', 'OverallCond_num', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', \\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'HeatingQC_num', \\\n", " 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Recon',\n", " 'ReconAfterSold', 'SoldImmediate'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(df_new), axis=1)\n", "df_new = pd.concat((df_new, XP), axis=1)\n", "\n", "df_new.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9726c026-cabc-3cab-99cd-8b2280410445" }, "source": [ "**>>>>>>>>>>>>>>>>End of arguable feature engineering>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d2991ab4-f1be-2b47-d272-cd3e0ed782e6" }, "source": [ "**>>>>>>>>>>>>>>>>A LASSO cross validation>>>>>>>>>>>>>>>>>>**\n", "\n", "### 1. Cross valication" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "31b7ed70-d3a6-e59d-ce11-f31c50975df2" }, "outputs": [], "source": [ "df_final = df_new.copy()\n", "\n", "X_train = df_final[:train.shape[0]]\n", "print(X_train.shape)\n", "y_train = np.log1p(train.SalePrice)\n", "print(y_train.shape)\n", "X_test = df_final[test.shape[0]+1:]\n", "print(X_test.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5d158d26-034c-b5f2-bcc5-31cd279ada93" }, "outputs": [], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "from sklearn.cross_validation import cross_val_score\n", "from sklearn.metrics import make_scorer, mean_squared_error\n", "\n", "def rmse_cv(model, X_train, y_train):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring=\"neg_mean_squared_error\", cv=10))\n", " return(rmse)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "96d46562-2ca1-16e6-2ce4-ef8876885e2c" }, "outputs": [], "source": [ "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result = pd.Series(cv_lasso, index = alphas)\n", "result.plot()\n", "result.min()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "2d94d354-788f-a36a-b41b-c07f3e8e5b95" }, "source": [ "### 2. Detect outliers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "75efcceb-2700-1e3e-780b-3fef87204575", "collapsed": true }, "outputs": [], "source": [ "lasso_test = Lasso(alpha=0.00005, max_iter=1000).fit(X_train, y_train)\n", "y_test = lasso_test.predict(X_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "75b4c4e5-07e8-8464-b8dc-1fd0ebd7e59c" }, "outputs": [], "source": [ "dfy = pd.DataFrame({\"id\": train.Id, \"y_real\": y_train, \"y_test\": y_test, \"residual\": y_test - y_train},\\\n", " columns=['id', 'y_real', 'y_test', 'residual'])\n", "\n", "outliers_id = np.array([524, 1299])\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a6d74693-b3d9-2475-7339-bfb37cead579" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='y_test', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 2], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4a1781db-be0b-9fcc-7d1c-b087f5f72b6a" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='residual', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 3], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "13a68d68-c7e5-6425-bd8a-0ecfdb1b0620" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "ax = plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d78f22d7-3f3e-3d0f-91c8-40e9dc753be7" }, "source": [ "# Two obvious outliers!!!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "de0189ac-95a0-283f-6d4f-af6238987adf" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id, axis=0)\n", "y_train = y_train.drop(outliers_id, axis=0)\n", "\n", "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result2 = pd.Series(cv_lasso, index = alphas)\n", "df_result = pd.concat((result, result2), axis=1)\n", "df_result.columns= ['original', 'without outliers']\n", "df_result.plot()\n", "# result.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "82f3505e-f64d-b1d7-c498-ae5e6f59ca42" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0996c5cf-8066-ecec-e10e-5712e7c10ca5" }, "outputs": [], "source": [ "lasso = Lasso(alpha=0.0005, max_iter=1000).fit(X_train, y_train)\n", "y_pred = lasso.predict(X_test)\n", "y_pred.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0aa0da9b-c67e-1090-4cec-0c87b20c6321" }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_pred)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"lasso_sol.csv\", index = False)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }