{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f3eda3fa-9ce9-8299-102f-c66be32694ca" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9f4ee799-5c75-ff87-af7f-25a076b25db0" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "00698292-2dc1-0e70-ad54-5ee7183ef378" }, "outputs": [], "source": [ "df = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'], test.loc[:, 'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "776d375a-d94a-f50e-213a-14f33d1ac42f" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Data Summary>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c7740eee-2bed-273b-349a-6b1aba09db05" }, "outputs": [], "source": [ "print(train.shape)\n", "print(test.shape)\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d5463fd4-0c5c-9733-4149-984001b1c547" }, "outputs": [], "source": [ "df.dtypes.value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2c7311c8-8ab7-3292-9406-7a2805c48a88" }, "outputs": [], "source": [ "object_feats = list(df.dtypes[df.dtypes == \"object\"].index)\n", "print(len(object_feats))\n", "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "92968e59-8c45-fd29-184e-2c7102f0b2e0" }, "outputs": [], "source": [ "int_feats = list(df.dtypes[df.dtypes == \"int64\"].index)\n", "print(len(int_feats))\n", "print(int_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c159c201-14a0-ed0e-f282-4c3f3186a223" }, "outputs": [], "source": [ "float_feats = list(df.dtypes[df.dtypes == \"float64\"].index)\n", "print(len(float_feats))\n", "print(float_feats)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "746d2da9-135b-3baa-1e61-30fc529b516a" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>Missing value processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a5f31a26-90ff-7eb3-7dcb-443809261143" }, "outputs": [], "source": [ "na_counts = train.isnull().sum(axis=0)\n", "print('number of features containing NA')\n", "print('train: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = test.isnull().sum(axis=0)\n", "print('test: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = df.isnull().sum(axis=0)\n", "print('all: ', na_counts.loc[na_counts > 0].shape[0])\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c4b61d6-01be-a1e9-2aa8-4b31ca76f148" }, "outputs": [], "source": [ "num_feats = int_feats + float_feats\n", "NA_num = [x for x in list(na_counts.loc[na_counts > 0].index) if x in num_feats]\n", "NA_obj = [x for x in list(na_counts.loc[na_counts > 0].index) if x in object_feats]\n", "print('number of numerical features with NA: ', len(NA_num), '\\nnumber of object features with NA: ', len(NA_obj))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "57e3652e-fdb6-75bf-151e-edd843ef4c01" }, "outputs": [], "source": [ "print(NA_num)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "203123b5-ded6-fd26-32f2-54fee75778dc" }, "source": [ "### 1. Numerical features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "aaf24724-1561-de5c-6c98-92863e7d0b28" }, "outputs": [], "source": [ "# Let's deal with numerical features first\n", "# We might process some object features in the mean time, if they are about the same properties\n", "# And we use a list to store the \"btw processed\" object features, if their NAs are completely eliminated\n", "NA_obj_btw = []\n", "# Also, we use a list to store the object features that are ordinal categorical\n", "ordianl_obj = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b8b8851b-9e6b-5974-9eec-c9abbbcb2ab8" }, "outputs": [], "source": [ "# LotFrontage\n", "mean_LotFrontage = df.groupby('BldgType').LotFrontage.mean()\n", "for x in list(mean_LotFrontage.index):\n", " df.loc[(df.LotFrontage.isnull()) & (df.BldgType==x), 'LotFrontage'] = mean_LotFrontage[x]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5415a7b8-7e13-92bb-0541-fa7fe09e7198" }, "outputs": [], "source": [ "# MasVnrArea\n", "df.loc[df.MasVnrType.isnull(), ['MasVnrType', 'MasVnrArea']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "faad129b-c3a8-51ea-5ecf-a6a11d7b79e7" }, "outputs": [], "source": [ "df.MasVnrType.value_counts(dropna=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "85248c3c-e3f8-24d1-0eae-7d723f1c83bf" }, "outputs": [], "source": [ "# We have to fill na correspondingly for both of them\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrType'] = 'None'\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrArea'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9144c837-c92d-976e-b061-26ef8259e45d" }, "outputs": [], "source": [ "# and there is one REAL NaN in MasVnrType, row 1150 shown above\n", "df.loc[df.MasVnrType.isnull(), 'MasVnrType'] = 'BrkFace'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0b153fe5-13f5-6a0c-f56f-edd467f28b4a" }, "outputs": [], "source": [ "NA_obj_btw.append('MasVnrType')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "75942d0b-25e7-4afb-c590-fd17d2a2ee8f" }, "outputs": [], "source": [ "# features related to basement\n", "base_feats_num = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']\n", "for x in base_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2cbae5ef-6de4-654d-a0df-16cb10d07a00" }, "outputs": [], "source": [ "# maybe it's better to do with all basement features together\n", "# features related to basement\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41dbf475-ea1c-28fd-8b85-5354f2735b67" }, "outputs": [], "source": [ "df.loc[df.BsmtFullBath.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1510655c-1aaf-5d90-4955-f781d11fa7b9" }, "outputs": [], "source": [ "# it's obvious that these two houses have no basement\n", "df.loc[df.BsmtFullBath.isnull(), base_feats_num] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ed042237-9e3f-fa46-230d-43891ae552f9" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType1.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4e9aa4da-e6e3-c79f-b944-b52d8481caf5" }, "outputs": [], "source": [ "# as in the data description, most NaN means no basement\n", "df.loc[df.BsmtCond.isnull(), base_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0820b020-4c97-490d-3168-ba6a71c50a8a" }, "outputs": [], "source": [ "# Maybe there are some REAL NaNs\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "832c61b0-fa0b-1534-72e0-faed199a732e" }, "outputs": [], "source": [ "df.loc[df.BsmtQual.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f032f02d-ca49-9d68-7665-f0212bcb9f44" }, "outputs": [], "source": [ "df.BsmtQual.mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "87e43bfe-4249-4005-9f46-2451c75f36f0", "collapsed": true }, "outputs": [], "source": [ "# yes the BsmtQual measures the height of the basement, and there are two NaNs\n", "df.loc[df.BsmtQual.isnull(), 'BsmtQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e8cc383a-43be-826c-1f73-8476fa4d75fc" }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d9fabf6a-f6c3-621d-09b5-40448bc6b4c3" }, "outputs": [], "source": [ "df['BsmtUnfinishRatio'] = df.BsmtUnfSF / df.TotalBsmtSF\n", "grouped = df.groupby('BsmtExposure')\n", "grouped['BsmtUnfinishRatio'].mean().sort_values()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e7667bd7-b61e-cf67-24da-4ad0226e6bb0", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), 'BsmtExposure'] = 'No'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "00a6a9ca-ae8d-3783-18d8-a6cb01c10877" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c4583d3f-185c-d8da-bf00-0fcc251bae5b" }, "outputs": [], "source": [ "grouped = df.groupby('BsmtFinType2')\n", "grouped['BsmtFinSF2'].mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e30097dc-c4bf-fa56-0481-514f0a6bfcbb", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'ALQ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "515cf6fd-d54a-d6e2-ca29-2bdd071b9865" }, "outputs": [], "source": [ "# Maybe there are some real NaNs\n", "base_feats = base_feats_num + base_feats_obj\n", "for x in base_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e041c94e-4636-c74e-95fc-c6595d8b792e" }, "outputs": [], "source": [ "base_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7b537d8b-223a-8229-8759-2253951c6007", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + ['BsmtQual', 'BsmtCond', 'BsmtFinType1']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3f6af005-3154-ff2e-d126-c8f31ff4cf26", "collapsed": true }, "outputs": [], "source": [ "# garage\n", "# again, it's better to deal with all garage features in the mean time\n", "garage_feats_num = ['GarageYrBlt', 'GarageCars', 'GarageArea']\n", "garage_feats_obj = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "944cce19-8f24-9433-0d39-aa552fc3715a" }, "outputs": [], "source": [ "for x in garage_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1e322b10-94c1-d7de-15e1-2086924a842b" }, "outputs": [], "source": [ "for x in garage_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6d2d0c5b-3e3d-ad01-e6e4-c93db0626cc7" }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), garage_feats_num+garage_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d0fcdb9d-a0dc-cada-83df-fbdb7c149e0e", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), ['GarageCars', 'GarageArea']] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a194b617-d143-ab81-2447-4cab80cdd46e" }, "outputs": [], "source": [ "df.loc[df.GarageType.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "efb155c7-2e87-4ff8-79d3-65c68b813eeb", "collapsed": true }, "outputs": [], "source": [ "# as in the data description, NaN means no garage\n", "df.loc[df.GarageType.isnull(), garage_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "14871a8b-4eeb-c0f0-3518-7deb3b6c2991" }, "outputs": [], "source": [ "df.loc[df.GarageQual.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d9e51b22-9b74-902e-9fe3-1171f480571a" }, "outputs": [], "source": [ "df.loc[df.GarageCars==1, ['GarageFinish', 'GarageQual', 'GarageCond']].mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "41bcf12c-9ac7-a175-14d1-5311a1a7cd0a", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==1), 'GarageFinish'] = 'Unf'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==1), 'GarageQual'] = 'TA'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==1), 'GarageCond'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "00a6c7b7-384d-8d05-93ac-d356d5749e1a", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==0), 'GarageFinish'] = 'Without'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==0), 'GarageQual'] = 'Without'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==0), 'GarageCond'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3e4bde7e-3b7a-d21e-c633-d63001bebeea", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageYrBlt.isnull(), 'GarageYrBlt'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "86c0f86a-1232-b80b-58aa-9f8dd22ead4b" }, "outputs": [], "source": [ "garage_feats = garage_feats_num + garage_feats_obj\n", "for x in garage_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "040d8c38-a02f-c446-4cd7-3f3d57875f9b", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + garage_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6541aecf-aeb9-3fa7-6a74-cc3ec299aed8" }, "outputs": [], "source": [ "# let's check the outcome of the missing values processing\n", "for x in NA_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9439261f-4d58-69fb-cb83-1f0d419ff0d5" }, "source": [ "### 2. Object features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ce78431f-0115-2c6b-0544-692135ba0f30" }, "outputs": [], "source": [ "# OK, not bad, let's move on to object features\n", "# let's have a look at what have been processed \"BTW\"\n", "NA_obj_btw" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dabee7f5-ce54-d619-1fed-927c52288414" }, "outputs": [], "source": [ "NA_obj_new = [x for x in NA_obj if x not in NA_obj_btw]\n", "print(NA_obj_new)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ecef91b2-d4f2-aeac-15ee-d80faffc7878" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3e3899f6-2dcd-bc5b-c642-90754c2e4a35" }, "outputs": [], "source": [ "# MSZoning\n", "print(df.MSZoning.mode())\n", "df.loc[df.MSZoning.isnull(), 'MSZoning'] = 'RL'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2e390cbc-4255-2581-8ee8-4169ede98bfa", "collapsed": true }, "outputs": [], "source": [ "# Alley\n", "# NaN means no alley\n", "df.loc[df.Alley.isnull(), 'Alley'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1417bcfc-2db7-2435-f529-e662ec97b02f" }, "outputs": [], "source": [ "# Utilities\n", "print(df.Utilities.mode())\n", "df.loc[df.Utilities.isnull(), 'Utilities'] = 'AllPub'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "261488b9-b65b-59fb-1883-8c2f3d9247c6" }, "outputs": [], "source": [ "# 'Exterior1st' and 'Exterior2nd'\n", "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1a223634-1b3b-15a1-42da-7761992e3ae3" }, "outputs": [], "source": [ "print(df[['Exterior1st', 'Exterior2nd']].mode())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bcd7487e-609b-e9ae-4b3b-6bb70ac99056", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']] = 'VinylSd'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bfaa3cd6-d741-5df0-3b82-d9318498dc33", "collapsed": true }, "outputs": [], "source": [ "# Electrical\n", "df.Electrical.mode()\n", "df.loc[df.Electrical.isnull(), 'Electrical'] = 'SBrkr'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "22446777-4aec-fefd-6667-785145aac2fd", "collapsed": true }, "outputs": [], "source": [ "# KitchenQual\n", "df.KitchenQual.mode()\n", "df.loc[df.KitchenQual.isnull(), 'KitchenQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0fec404a-99a3-2104-00f2-9f977f0922b8", "collapsed": true }, "outputs": [], "source": [ "# Functional\n", "df.Functional.mode()\n", "df.loc[df.Functional.isnull(), 'Functional'] = 'Typ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5e92f069-a5a9-10b2-a41c-9f1dd0be4870", "collapsed": true }, "outputs": [], "source": [ "# FireplaceQu\n", "# NaN means no fireplace\n", "df.loc[df.FireplaceQu.isnull(), 'FireplaceQu'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5bb105f3-e388-7fe6-439a-25d10d62b002", "collapsed": true }, "outputs": [], "source": [ "# PoolQC\n", "# NaN means no pool\n", "df.loc[df.PoolQC.isnull(), 'PoolQC'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "93303021-1151-3215-dd47-904eaf58cb39", "collapsed": true }, "outputs": [], "source": [ "# Fence\n", "# NaN means no fence\n", "df.loc[df.Fence.isnull(), 'Fence'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "63ffd8c3-a01f-0ef0-829a-d861fde985e7", "collapsed": true }, "outputs": [], "source": [ "# MiscFeature\n", "# NaN means no MiscFeature\n", "df.loc[df.MiscFeature.isnull(), 'MiscFeature'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d3f72499-5aec-8e1d-bcea-ec4918376cde", "collapsed": true }, "outputs": [], "source": [ "# SaleType\n", "df.SaleType.mode()\n", "df.loc[df.SaleType.isnull(), 'SaleType'] = 'WD'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "566e6883-9083-8263-647b-765fa932f5f9" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e4007aed-ac73-7253-5fd7-01e95cf34da9" }, "outputs": [], "source": [ "# let's check if there is still NAs\n", "na_counts = df.isnull().sum(axis=0)\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6bedd907-f9f6-ca33-59d0-32209f67e54b", "collapsed": true }, "outputs": [], "source": [ "df.drop(['BsmtUnfinishRatio'], inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "301c4eb0-d49a-659a-27e0-5ea6f4865076" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>End of Missing Value Processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "dd292f34-dba4-7488-282c-6733a7319340" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>Transform ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b20ecb0f-9efb-fea0-41f3-72d2aa3789f6" }, "outputs": [], "source": [ "print('Before', len(object_feats))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ca6c24ba-f177-7c8c-46e7-9a2b8311d430" }, "outputs": [], "source": [ "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e67b95a9-87c9-0e91-b426-e33518d89879", "collapsed": true }, "outputs": [], "source": [ "ordinal_words = ['Ex', 'Gd', 'TA', 'Fa', 'Po']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a3c4cefd-0d7c-9976-0bd3-658d229123fb", "collapsed": true }, "outputs": [], "source": [ "ordinal_feats = []\n", "for x in object_feats:\n", " flag = 0\n", " for i in ordinal_words:\n", " if df.loc[df[x].str.contains(i, case=True), x].shape[0] > 0:\n", " flag = flag + 1\n", " if flag >= 2:\n", " ordinal_feats.append(x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "68391617-ddd2-0f10-f0e4-bebdca5f6756" }, "outputs": [], "source": [ "print(ordinal_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5e948b5f-adc6-3c39-cb13-e89e070ceeda" }, "outputs": [], "source": [ "for x in ordinal_feats:\n", " print(x, df[x].unique(), len(df[x].unique()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "701276ff-529e-7bae-104d-98c3f46520e5", "collapsed": true }, "outputs": [], "source": [ "ordinal_dict = {'Ex':5, 'Gd':4, 'TA':3, 'Fa':2, 'Po':1, 'Without':0}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c033c2b2-e1ff-e3cb-2840-edbdce55ddf1", "collapsed": true }, "outputs": [], "source": [ "ordinal_num_feats = []\n", "for x in ordinal_feats:\n", " df[x+'_num'] = df[x].map(ordinal_dict)\n", " ordinal_num_feats.append(x+'_num')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6867ec59-93f9-3671-8227-c0937a768662", "collapsed": true }, "outputs": [], "source": [ "df.drop(ordinal_feats, axis=1, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e1013231-e5db-916a-4759-4a44f241803c", "collapsed": true }, "outputs": [], "source": [ "ordinal_obj_feats = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'Utilities']\n", "\n", "df['Functional_num'] = df.Functional.replace({'Typ': 6,\n", " 'Min1': 5,\n", " 'Min2': 5,\n", " 'Mod': 4,\n", " 'Maj1': 3,\n", " 'Maj2': 3,\n", " 'Sev': 2,\n", " 'Sal': 1})\n", "df['PavedDrive_num'] = df.PavedDrive.replace({'Y': 3, 'P': 2, 'N': 1})\n", "df['CentralAir_num'] = df.CentralAir.replace({'Y': 1, 'N': 0})\n", "df['Fence_num'] = df.CentralAir.replace({'GdPrv': 2, 'GdWo': 2, 'MnPrv': 1, 'MnWw': 1, 'NoFence': 0})\n", "df['Utilities_num'] = df.Utilities.replace({'AllPub': 1, 'NoSewr': 0, 'NoSeWa': 0, 'ELO': 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "040c9fe9-43a8-724b-5e7b-4a75a4e9b8e3", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []\n", "toDrop_feats = toDrop_feats + ordinal_obj_feats" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bd3be15d-caf3-9a17-0d46-14f3e157e812" }, "outputs": [], "source": [ "df.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c4fd74e3-00c9-8a76-0730-d3a5c24b5682" }, "source": [ "**>>>>>>>>>>>>>>>>End of transformation of ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "1376a0ce-9bb5-babf-49ec-b676f7fbe144" }, "source": [ "**>>>>>>>>>>>>>>>>Some arguable feature engineering>>>>>>>>>>>>>>>>>>**\n", "\n", "**For any cells in this part, if you don't want it, just mark or delete the whole well, it won't affect cells after.**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "ba302551-4c40-02e7-93a8-a85cb731c3fc" }, "source": [ "### 1. Try to make quality features and some categorical features more \"significant\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "455691b6-cc75-0c76-dfa0-c592fe54f493", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3523b338-e3fd-c750-865a-29762b34e8d6", "collapsed": true }, "outputs": [], "source": [ "# some maybe newer\n", "df['newer_dwelling'] = df.MSSubClass.replace({20: 1, 30: 0, 40: 0, 45: 0, 50: 0, 60: 1, 70: 0, 75: 0,\\\n", " 80: 0, 85: 0, 90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})\n", "\n", "# but we still keep the original feature and transform it into categorical\n", "map_MSS = {x: 'Subclass_'+str(x) for x in df.MSSubClass.unique()}\n", "df['MSSubClass'] = df.MSSubClass.replace(map_MSS)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "42b7b0bc-0fca-1fe8-60ca-9da117ca957a" }, "outputs": [], "source": [ "# it maybe helpful to transform some quality style features into binary\n", "quality_feats = ['OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num',\\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num']\n", "toDrop_feats = toDrop_feats + quality_feats\n", "\n", "all_data = df\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual_num.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual_num.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond_num.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond_num.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond_num.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond_num.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual_num.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual_num.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond_num.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond_num.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual_num.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual_num.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "df_qual = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "df = pd.concat((df, df_qual), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "74fd138e-f5b5-ce88-82b6-345fea7f4dd8", "collapsed": true }, "outputs": [], "source": [ "# for some categorical features, certain levels may imply better quality\n", "toDrop_feats = toDrop_feats + ['MasVnrType', 'SaleCondition', 'Neighborhood']\n", "\n", "map_Mas = {'BrkCmn': 1, 'BrkFace': 1, 'CBlock': 1, 'Stone': 1, 'None': 0}\n", "MasVnrType_Any = all_data.MasVnrType.replace(map_Mas)\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "map_Sale = {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0}\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace(map_Sale)\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "neigh_good_feats = ['NridgHt', 'Crawfor', 'StoneBr', 'Somerst', 'NoRidge']\n", "df['Neighborhood_good'] = 0\n", "df.loc[df.Neighborhood.isin(neigh_good_feats), 'Neighborhood_good'] = 1\n", "\n", "df = pd.concat((df, MasVnrType_Any, SaleCondition_PriceDown), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "590eaf63-a50d-eae5-0a43-51920ed8239e", "collapsed": true }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "df['season'] = df.MoSold.replace({1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "715c3e9c-0cda-bb53-872b-127a3e43dbf2" }, "outputs": [], "source": [ "# Numer month is not significant, it maybe helpful to transform them into object feature\n", "map_Mo = {1: 'Yan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Avg', 9: 'Sep', 10: 'Oct', \\\n", " 11: 'Nov', 12: 'Dec'}\n", "df = df.replace({'MoSold': map_Mo})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f0682957-0156-aab1-c8b5-539cf2e4b0b8", "collapsed": true }, "outputs": [], "source": [ "# some features about years\n", "# sold at the same year as bulit\n", "df['SoldImmediate'] = 0\n", "df.loc[(df.YrSold == df.YearBuilt), 'SoldImmediate'] = 1\n", "\n", "# reconstructed since first built\n", "df['Recon'] = 0\n", "df.loc[(df.YearBuilt < df.YearRemodAdd), 'Recon'] = 1\n", "\n", "# reconstructed after sold\n", "df['ReconAfterSold'] = 0\n", "df.loc[(df.YrSold < df.YearRemodAdd), 'ReconAfterSold'] = 1\n", "\n", "# reconstructed the same year as sold\n", "df['ReconEqualSold'] = 0\n", "df.loc[(df.YrSold == df.YearRemodAdd), 'ReconEqualSold'] = 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "80b68f08-0396-8329-b808-19b75585225c", "collapsed": true }, "outputs": [], "source": [ "# Years are too much, it maybe helpful to devide them into groups, and delete the original ones\n", "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "df.YearBuilt = df.YearBuilt.map(year_map)\n", "df.YearRemodAdd = df.YearRemodAdd.map(year_map)\n", "df.GarageYrBlt = df.GarageYrBlt.map(year_map)\n", "df.loc[df.GarageYrBlt==0, 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "31ec582a-c863-716f-3172-230050ad8433" }, "outputs": [], "source": [ "print(len(toDrop_feats), df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c626f838-0b3b-6dfb-1dd3-dade1992ab9c", "collapsed": true }, "outputs": [], "source": [ "t = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'MasVnrType', 'SaleCondition', 'Neighborhood']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c6de67f7-63fc-b896-5445-2172dabe0e80" }, "outputs": [], "source": [ "[x for x in t if x not in toDrop_feats]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2e6ee97c-5c19-9b79-57dc-90e0137d676a", "collapsed": true }, "outputs": [], "source": [ "temp = ['MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'Foundation', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating', 'CentralAir', 'Electrical', 'X1stFlrSF', 'X2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', 'X3SsnPorch', 'ScreenPorch', 'PoolArea', 'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType', 'SaleCondition', 'ExterQual_num', 'ExterCond_num', 'BsmtQual_num', 'BsmtCond_num', 'HeatingQC_num', 'KitchenQual_num', 'FireplaceQu_num', 'GarageQual_num', 'GarageCond_num', 'PoolQC_num', 'Functional_num', 'PavedDrive_num', 'CentralAir_num', 'Fence_num', 'Utilities_num', 'newer_dwelling', 'overall_poor_qu', 'overall_good_qu', 'overall_poor_cond', 'overall_good_cond', 'exter_poor_qu', 'exter_good_qu', 'exter_poor_cond', 'exter_good_cond', 'bsmt_poor_cond', 'bsmt_good_cond', 'garage_poor_qu', 'garage_good_qu', 'garage_poor_cond', 'garage_good_cond', 'kitchen_poor_qu', 'kitchen_good_qu', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Neighborhood_good', 'season', 'SoldImmediate', 'Recon', 'ReconAfterSold', 'ReconEqualSold']" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "55179edc-fce0-5047-cf75-c0eac91d13a7" }, "source": [ "### 2. Do SVC for some features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8920af96-c778-f1a5-4684-7e068838d3ca" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc])\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)\n", "\n", "price_category = pd.DataFrame(np.zeros((df.shape[0],1)), columns=['pc'], index=df.index)\n", "X_t = pd.get_dummies(df.loc[:, columns_for_pc])\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "\n", "toDrop_feats = toDrop_feats + columns_for_pc\n", "df = pd.concat((df, price_category), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8d99c7b0-545d-dcb6-9e40-82b5d95b7f34" }, "source": [ "### 3. Scale the numerical data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fb781d18-8076-bfe6-52f8-31428721f7d6", "collapsed": true }, "outputs": [], "source": [ "numeric_feats = df.dtypes[df.dtypes != \"object\"].index\n", "t = df[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "df[use_max_scater] = df[use_max_scater]/df[use_max_scater].max()\n", "df[use_95_scater] = df[use_95_scater]/df[use_95_scater].quantile(.95)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e4f10b6b-6cdd-4088-1533-88897c986c60" }, "source": [ "### 4. Log skewed data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f48b6e7c-9c35-8bd2-0dfa-c66694e2cbbf", "collapsed": true }, "outputs": [], "source": [ "df_new = df.copy()\n", "from scipy.stats import skew\n", "numeric_feats = list(df_new.dtypes[df_new.dtypes != \"object\"].index)\n", "feat_skewness = df_new[numeric_feats].apply(lambda x: skew(x.dropna()))\n", "skewed_feats = list(feat_skewness[feat_skewness > 0.75].index)\n", "df_new[skewed_feats] = np.log1p(df_new[skewed_feats])\n", "\n", "df = df_new.copy()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "486d5a78-682e-6465-0ce3-aeff121fe313" }, "source": [ "### 5. Drop some features that have been used to create new features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "62f5262f-7e30-821a-5365-d2f9b7459b26" }, "outputs": [], "source": [ "# print(len(toDrop_feats), toDrop_feats)\n", "# df.drop(toDrop_feats, inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7174cfab-edfc-cca5-38d9-b594bbe11dd7" }, "source": [ "### 6. Get dummy features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "78c5f7f9-515c-641b-6e57-a1d7511a27c2" }, "outputs": [], "source": [ "df_new = pd.get_dummies(df)\n", "print(df.shape, df_new.shape)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a64fdefc-4411-b874-9adf-6fd46300ec3e" }, "source": [ "### 7. Create interact features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "767d47a7-4157-a2d9-2f56-c232bc230f51" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " t = chain(df_qual.axes[1].get_values(), \n", " ['OverallQual_num', 'OverallCond_num', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', \\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'HeatingQC_num', \\\n", " 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Recon',\n", " 'ReconAfterSold', 'SoldImmediate'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(df_new), axis=1)\n", "df_new = pd.concat((df_new, XP), axis=1)\n", "\n", "df_new.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0065a9b1-500a-d372-4e58-8894bdb55aea" }, "source": [ "**>>>>>>>>>>>>>>>>End of arguable feature engineering>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "86d68bee-eadf-1078-df19-665a07bc2bdd" }, "source": [ "**>>>>>>>>>>>>>>>>A LASSO cross validation>>>>>>>>>>>>>>>>>>**\n", "\n", "### 1. Cross valication" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "aae6b4cc-af62-0aed-cba6-508e64b9c849" }, "outputs": [], "source": [ "df_final = df_new.copy()\n", "\n", "X_train = df_final[:train.shape[0]]\n", "print(X_train.shape)\n", "y_train = np.log1p(train.SalePrice)\n", "print(y_train.shape)\n", "X_test = df_final[test.shape[0]+1:]\n", "print(X_test.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e383e5f0-cc5e-683f-0df2-a131aa39f959", "collapsed": true }, "outputs": [], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "from sklearn.cross_validation import cross_val_score\n", "from sklearn.metrics import make_scorer, mean_squared_error\n", "\n", "def rmse_cv(model, X_train, y_train):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring=\"neg_mean_squared_error\", cv=10))\n", " return(rmse)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eb8281bb-6872-d6ad-8d0e-5b260785f4a1" }, "outputs": [], "source": [ "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result = pd.Series(cv_lasso, index = alphas)\n", "result.plot()\n", "result.min()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c90f5710-15db-82c0-8ac4-2411043eae4e" }, "source": [ "### 2. Detect outliers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2a4024ba-b7ce-c946-fb3c-b3075d00bda4", "collapsed": true }, "outputs": [], "source": [ "lasso_test = Lasso(alpha=0.00005, max_iter=1000).fit(X_train, y_train)\n", "y_test = lasso_test.predict(X_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b32e751d-f221-8ac0-218c-3a336f1f0a66" }, "outputs": [], "source": [ "dfy = pd.DataFrame({\"id\": train.Id, \"y_real\": y_train, \"y_test\": y_test, \"residual\": y_test - y_train},\\\n", " columns=['id', 'y_real', 'y_test', 'residual'])\n", "\n", "outliers_id = np.array([524, 1299])\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0a0507d5-68e3-efdd-7168-cb6b22d4df81" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='y_test', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 2], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ba85d12e-781c-a8e5-15f7-83ee5685524e" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='residual', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 3], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "18ec5b41-24ca-5693-086d-02fdb21eb558" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "ax = plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9b11adc7-d6fe-2813-2214-5e8a49193358" }, "source": [ "# Two obvious outliers!!!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9785dbb3-9d86-89c6-6607-454add6fb3fa" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id, axis=0)\n", "y_train = y_train.drop(outliers_id, axis=0)\n", "\n", "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result2 = pd.Series(cv_lasso, index = alphas)\n", "df_result = pd.concat((result, result2), axis=1)\n", "df_result.columns= ['original', 'without outliers']\n", "df_result.plot()\n", "# result.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2d194699-00d3-ff21-6f60-e5cca5ddaf3e" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6b4d327e-0976-201e-a314-4b5e9b8646c5" }, "outputs": [], "source": [ "lasso = Lasso(alpha=0.00075, max_iter=1000).fit(X_train, y_train)\n", "y_pred = lasso.predict(X_test)\n", "y_pred.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1578ec73-8024-4b30-ed40-a053f3c13918" }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_pred)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"lasso_sol.csv\", index = False)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }