{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eee25f29-74ec-2a89-7684-350b3f66098c" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6f6843e9-f022-5ab8-16ed-9a4538e64f98" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8354da87-2e2b-8233-c015-26c0b9669ce2" }, "outputs": [], "source": [ "df = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'], test.loc[:, 'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "843baf7c-dbe7-50f0-7dac-d0fff1be487c" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Data Summary>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ff74d3b2-1f00-0e38-2792-10097f02b1c1" }, "outputs": [], "source": [ "print(train.shape)\n", "print(test.shape)\n", "print(df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1065abbd-00ac-93b4-eb28-96a7d13c13c3" }, "outputs": [], "source": [ "df.dtypes.value_counts()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e58d28cc-4175-cc56-fd49-2ba46e598db4" }, "outputs": [], "source": [ "object_feats = list(df.dtypes[df.dtypes == \"object\"].index)\n", "print(len(object_feats))\n", "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "afa18796-c92a-5810-cf50-5827f6b34f84" }, "outputs": [], "source": [ "int_feats = list(df.dtypes[df.dtypes == \"int64\"].index)\n", "print(len(int_feats))\n", "print(int_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2ad326c3-adb0-c0cd-d9f3-f544e795b70c" }, "outputs": [], "source": [ "float_feats = list(df.dtypes[df.dtypes == \"float64\"].index)\n", "print(len(float_feats))\n", "print(float_feats)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c8c9197f-816e-c8d1-ba99-17d430efc335" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>>>Missing value processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e2027191-67ec-6b27-b8f9-252c8038baa4" }, "outputs": [], "source": [ "na_counts = train.isnull().sum(axis=0)\n", "print('number of features containing NA')\n", "print('train: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = test.isnull().sum(axis=0)\n", "print('test: ', na_counts.loc[na_counts > 0].shape[0])\n", "na_counts = df.isnull().sum(axis=0)\n", "print('all: ', na_counts.loc[na_counts > 0].shape[0])\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d9debdef-c4aa-9c36-ed5a-7ed54c73ffc9" }, "outputs": [], "source": [ "num_feats = int_feats + float_feats\n", "NA_num = [x for x in list(na_counts.loc[na_counts > 0].index) if x in num_feats]\n", "NA_obj = [x for x in list(na_counts.loc[na_counts > 0].index) if x in object_feats]\n", "print('number of numerical features with NA: ', len(NA_num), '\\nnumber of object features with NA: ', len(NA_obj))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8ffe14aa-c190-98ac-d696-02068e758ba9" }, "outputs": [], "source": [ "print(NA_num)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7593ddae-55a9-953b-c345-66dd64f34b04" }, "source": [ "### 1. Numerical features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d0c408d0-9039-992c-cbe2-3f0d2c6c70d6", "collapsed": true }, "outputs": [], "source": [ "# Let's deal with numerical features first\n", "# We might process some object features in the mean time, if they are about the same properties\n", "# And we use a list to store the \"btw processed\" object features, if their NAs are completely eliminated\n", "NA_obj_btw = []\n", "# Also, we use a list to store the object features that are ordinal categorical\n", "ordianl_obj = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9084ad2c-8ffe-257b-8bda-44d92a4c0a83" }, "outputs": [], "source": [ "# LotFrontage\n", "mean_LotFrontage = df.groupby('BldgType').LotFrontage.mean()\n", "for x in list(mean_LotFrontage.index):\n", " df.loc[(df.LotFrontage.isnull()) & (df.BldgType==x), 'LotFrontage'] = mean_LotFrontage[x]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fde5269e-e71b-0c32-3104-1299f40c8427" }, "outputs": [], "source": [ "# MasVnrArea\n", "df.loc[df.MasVnrType.isnull(), ['MasVnrType', 'MasVnrArea']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2a18135b-62fa-7744-ee2f-4c5a3f3c09ee" }, "outputs": [], "source": [ "df.MasVnrType.value_counts(dropna=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "afb3bd9d-4ce2-122a-c5bb-1c3a0989ce1b", "collapsed": true }, "outputs": [], "source": [ "# We have to fill na correspondingly for both of them\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrType'] = 'None'\n", "df.loc[df.MasVnrArea.isnull(), 'MasVnrArea'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ccdbd998-3328-40fe-ca50-0613e8252504", "collapsed": true }, "outputs": [], "source": [ "# and there is one REAL NaN in MasVnrType, row 1150 shown above\n", "df.loc[df.MasVnrType.isnull(), 'MasVnrType'] = 'BrkFace'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7c331251-6829-0f19-d9e5-8148f4569f94", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw.append('MasVnrType')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1780a09a-1c42-3bbb-b0fa-c899153e20b7" }, "outputs": [], "source": [ "# features related to basement\n", "base_feats_num = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']\n", "for x in base_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "57bd9ca1-702d-cead-8723-bc4cd012db18" }, "outputs": [], "source": [ "# maybe it's better to do with all basement features together\n", "# features related to basement\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a71ce716-7a70-1043-1f68-3e35f2b2ed7b" }, "outputs": [], "source": [ "df.loc[df.BsmtFullBath.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "093593ac-1f65-4aaa-c2a2-a6f750cb12ad", "collapsed": true }, "outputs": [], "source": [ "# it's obvious that these two houses have no basement\n", "df.loc[df.BsmtFullBath.isnull(), base_feats_num] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1ee7f1bc-bc7a-4e12-aa21-59b8af54108f" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType1.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "08d02b5b-ef74-4c70-48b3-489184a91576", "collapsed": true }, "outputs": [], "source": [ "# as in the data description, most NaN means no basement\n", "df.loc[df.BsmtCond.isnull(), base_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e8cd0679-9953-373c-cd13-ed9a87e26930" }, "outputs": [], "source": [ "# Maybe there are some REAL NaNs\n", "base_feats_obj = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']\n", "for x in base_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ec3c4015-5a84-0cd5-1345-8b3a8e8cb491" }, "outputs": [], "source": [ "df.loc[df.BsmtQual.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fbb0db68-999b-f77b-6ae1-14cb4233a914" }, "outputs": [], "source": [ "df.BsmtQual.mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "33d4cc1d-2d05-5f1b-27e1-46cc0cf99df9", "collapsed": true }, "outputs": [], "source": [ "# yes the BsmtQual measures the height of the basement, and there are two NaNs\n", "df.loc[df.BsmtQual.isnull(), 'BsmtQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9da1b2fc-ecea-1fd9-6cf8-57bfa88b9355" }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), base_feats_num+base_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2bdd0c05-7948-c361-9bbb-87f8898ce3de" }, "outputs": [], "source": [ "df['BsmtUnfinishRatio'] = df.BsmtUnfSF / df.TotalBsmtSF\n", "grouped = df.groupby('BsmtExposure')\n", "grouped['BsmtUnfinishRatio'].mean().sort_values()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "97ee2b1a-24ee-21ea-6d75-3a11a821e8db", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtExposure.isnull(), 'BsmtExposure'] = 'No'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bb28ad5e-f22f-b8c6-fdc3-238f0c40c2ee" }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), base_feats_num+base_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9910bcd0-1700-7dbf-d484-c2beea80390f" }, "outputs": [], "source": [ "grouped = df.groupby('BsmtFinType2')\n", "grouped['BsmtFinSF2'].mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "dbad950d-a598-356c-7e15-b51f800e5ea1", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.BsmtFinType2.isnull(), 'BsmtFinType2'] = 'ALQ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c77c0370-d922-a8af-9d2c-882dc34e1594" }, "outputs": [], "source": [ "# Maybe there are some real NaNs\n", "base_feats = base_feats_num + base_feats_obj\n", "for x in base_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bb0a7702-97e1-9208-1536-3eebdfc3b866" }, "outputs": [], "source": [ "base_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fe8ee130-e38e-f1f6-b512-ee079fc949e9", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + ['BsmtQual', 'BsmtCond', 'BsmtFinType1']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1687cf47-7cd7-992a-3bc4-ed00d85326fc", "collapsed": true }, "outputs": [], "source": [ "# garage\n", "# again, it's better to deal with all garage features in the mean time\n", "garage_feats_num = ['GarageYrBlt', 'GarageCars', 'GarageArea']\n", "garage_feats_obj = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d50c9df5-a1d0-bc4c-e0fe-23e4e3f8a1eb" }, "outputs": [], "source": [ "for x in garage_feats_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "968e7876-e385-1aa1-f8a8-ece9562b5ef3" }, "outputs": [], "source": [ "for x in garage_feats_obj:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "90bfa5b7-e343-e588-db75-4e8ad06c40bb" }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), garage_feats_num+garage_feats_obj]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "77f05da3-6e38-3fd7-139e-aa01ca7a7c83", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageCars.isnull(), ['GarageCars', 'GarageArea']] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e809a510-2b52-202f-55c3-18fce3b22dbe" }, "outputs": [], "source": [ "df.loc[df.GarageType.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "67366cfc-4545-beb1-520f-8ec827d8271a", "collapsed": true }, "outputs": [], "source": [ "# as in the data description, NaN means no garage\n", "df.loc[df.GarageType.isnull(), garage_feats_obj] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "521ae83a-aecf-b213-8cba-6ed36fb71ab5" }, "outputs": [], "source": [ "df.loc[df.GarageQual.isnull(), garage_feats_num+garage_feats_obj].head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2e851098-fd83-2192-83cd-7e66ffed1fcc" }, "outputs": [], "source": [ "df.loc[df.GarageCars==1, ['GarageFinish', 'GarageQual', 'GarageCond']].mode()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "aafbaaf5-a2e4-2e6a-70af-4fdfaae4ed86", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==1), 'GarageFinish'] = 'Unf'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==1), 'GarageQual'] = 'TA'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==1), 'GarageCond'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "836bc009-69fe-227d-05bc-cc1462aa57ee", "collapsed": true }, "outputs": [], "source": [ "df.loc[(df.GarageFinish.isnull())&(df.GarageCars==0), 'GarageFinish'] = 'Without'\n", "df.loc[(df.GarageQual.isnull())&(df.GarageCars==0), 'GarageQual'] = 'Without'\n", "df.loc[(df.GarageCond.isnull())&(df.GarageCars==0), 'GarageCond'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2a2291ce-69c4-6643-90e9-17531b3aef93", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.GarageYrBlt.isnull(), 'GarageYrBlt'] = 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a03c23c9-7764-3d8c-3113-907a5bd7ed78" }, "outputs": [], "source": [ "garage_feats = garage_feats_num + garage_feats_obj\n", "for x in garage_feats:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f5f0eaca-bd09-9c00-ab1d-817380849a5b", "collapsed": true }, "outputs": [], "source": [ "NA_obj_btw = NA_obj_btw + garage_feats_obj" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9a3da830-5b52-e80c-d5ef-dc710eeb6fba" }, "outputs": [], "source": [ "# let's check the outcome of the missing values processing\n", "for x in NA_num:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "2ce97def-b55a-0ed1-1385-930b94e56295" }, "source": [ "### 2. Object features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6a327dc5-809e-4622-36f8-9045548cff63" }, "outputs": [], "source": [ "# OK, not bad, let's move on to object features\n", "# let's have a look at what have been processed \"BTW\"\n", "NA_obj_btw" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bd98f7f8-9872-d905-5fad-ba0a812d487b" }, "outputs": [], "source": [ "NA_obj_new = [x for x in NA_obj if x not in NA_obj_btw]\n", "print(NA_obj_new)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f6b86204-d631-c821-b922-ac020f676b70" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fcdaf430-5ed5-75fd-c819-0d297340f0f5" }, "outputs": [], "source": [ "# MSZoning\n", "print(df.MSZoning.mode())\n", "df.loc[df.MSZoning.isnull(), 'MSZoning'] = 'RL'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6c63a78f-92d6-1fa3-72dd-9f8b273147c9", "collapsed": true }, "outputs": [], "source": [ "# Alley\n", "# NaN means no alley\n", "df.loc[df.Alley.isnull(), 'Alley'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "593c55b7-ec70-7c1f-24c8-25bcb5ba0e1b" }, "outputs": [], "source": [ "# Utilities\n", "print(df.Utilities.mode())\n", "df.loc[df.Utilities.isnull(), 'Utilities'] = 'AllPub'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eeeddf27-94c0-70e5-7026-b87edd7cc8e6" }, "outputs": [], "source": [ "# 'Exterior1st' and 'Exterior2nd'\n", "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8f8145c5-3ee5-d6bc-7c1e-f1b027f839a4" }, "outputs": [], "source": [ "print(df[['Exterior1st', 'Exterior2nd']].mode())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "49e3f2a0-dfc2-7868-1117-48d0eaa30f14", "collapsed": true }, "outputs": [], "source": [ "df.loc[df.Exterior1st.isnull(), ['Exterior1st', 'Exterior2nd']] = 'VinylSd'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bd595759-2d76-ce26-c081-73f65f09972b", "collapsed": true }, "outputs": [], "source": [ "# Electrical\n", "df.Electrical.mode()\n", "df.loc[df.Electrical.isnull(), 'Electrical'] = 'SBrkr'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8ecd8ec7-09c5-51e4-7267-9054e87b6f1f", "collapsed": true }, "outputs": [], "source": [ "# KitchenQual\n", "df.KitchenQual.mode()\n", "df.loc[df.KitchenQual.isnull(), 'KitchenQual'] = 'TA'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b08f0821-f396-df96-8809-cf68e6f01f66", "collapsed": true }, "outputs": [], "source": [ "# Functional\n", "df.Functional.mode()\n", "df.loc[df.Functional.isnull(), 'Functional'] = 'Typ'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "884300fb-edc3-dba2-b261-e57f4869c979", "collapsed": true }, "outputs": [], "source": [ "# FireplaceQu\n", "# NaN means no fireplace\n", "df.loc[df.FireplaceQu.isnull(), 'FireplaceQu'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bf50c877-f660-ef36-3b58-a10663a58df1", "collapsed": true }, "outputs": [], "source": [ "# PoolQC\n", "# NaN means no pool\n", "df.loc[df.PoolQC.isnull(), 'PoolQC'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "138440ab-9744-18f1-66f1-61cffa410cb2", "collapsed": true }, "outputs": [], "source": [ "# Fence\n", "# NaN means no fence\n", "df.loc[df.Fence.isnull(), 'Fence'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7bbd7a7f-44b6-5b47-9d0e-3a0d788172cf", "collapsed": true }, "outputs": [], "source": [ "# MiscFeature\n", "# NaN means no MiscFeature\n", "df.loc[df.MiscFeature.isnull(), 'MiscFeature'] = 'Without'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a2678e19-edcd-6f25-9f9a-1cd9ca4e6408", "collapsed": true }, "outputs": [], "source": [ "# SaleType\n", "df.SaleType.mode()\n", "df.loc[df.SaleType.isnull(), 'SaleType'] = 'WD'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8e0d46dc-7d05-07f1-74ce-775b5d8373bf" }, "outputs": [], "source": [ "for x in NA_obj_new:\n", " print(x, df.loc[df[x].isnull(), :].shape[0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8574b025-6a15-d6a3-cfb9-3de927c9c266" }, "outputs": [], "source": [ "# let's check if there is still NAs\n", "na_counts = df.isnull().sum(axis=0)\n", "print(na_counts.loc[na_counts > 0])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8950d367-2ca4-5752-e351-f8a7c66e9aa8", "collapsed": true }, "outputs": [], "source": [ "df.drop(['BsmtUnfinishRatio'], inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "38968095-8f5c-8708-376b-e3e9c5691b7d" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>End of Missing Value Processing>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0c8bee32-7d16-edae-1937-038572c91098" }, "source": [ "**>>>>>>>>>>>>>>>>>>>>>>>>>>Transform ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cae5db06-c91a-1f3b-adb4-adc273723059" }, "outputs": [], "source": [ "print('Before', len(object_feats))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "214c59d1-dc9b-ddfb-5a69-5e939e7c6a0d" }, "outputs": [], "source": [ "print(object_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "51a8b293-97f4-bf5f-7090-58df9961b9ed", "collapsed": true }, "outputs": [], "source": [ "ordinal_words = ['Ex', 'Gd', 'TA', 'Fa', 'Po']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "adea696a-44f4-7a70-d5a0-a4e86e877e3c", "collapsed": true }, "outputs": [], "source": [ "ordinal_feats = []\n", "for x in object_feats:\n", " flag = 0\n", " for i in ordinal_words:\n", " if df.loc[df[x].str.contains(i, case=True), x].shape[0] > 0:\n", " flag = flag + 1\n", " if flag >= 2:\n", " ordinal_feats.append(x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e9d20a43-0200-c194-a8b7-ebcfa6eeb232" }, "outputs": [], "source": [ "print(ordinal_feats)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c15c7715-d126-e78f-1a83-507c3e316f16" }, "outputs": [], "source": [ "for x in ordinal_feats:\n", " print(x, df[x].unique(), len(df[x].unique()))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c717d33c-ee05-500d-ac7d-42fcd3dec715", "collapsed": true }, "outputs": [], "source": [ "ordinal_dict = {'Ex':5, 'Gd':4, 'TA':3, 'Fa':2, 'Po':1, 'Without':0}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1766c3d8-6796-4f13-9a28-2d88e70e8e18", "collapsed": true }, "outputs": [], "source": [ "ordinal_num_feats = []\n", "for x in ordinal_feats:\n", " df[x+'_num'] = df[x].map(ordinal_dict)\n", " ordinal_num_feats.append(x+'_num')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4dac4f53-2124-a169-f386-83ceae84be5a", "collapsed": true }, "outputs": [], "source": [ "df.drop(ordinal_feats, axis=1, inplace=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9406fcaa-c118-c360-e7d6-2aa37b66f685", "collapsed": true }, "outputs": [], "source": [ "ordinal_obj_feats = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'Utilities']\n", "\n", "df['Functional_num'] = df.Functional.replace({'Typ': 6,\n", " 'Min1': 5,\n", " 'Min2': 5,\n", " 'Mod': 4,\n", " 'Maj1': 3,\n", " 'Maj2': 3,\n", " 'Sev': 2,\n", " 'Sal': 1})\n", "df['PavedDrive_num'] = df.PavedDrive.replace({'Y': 3, 'P': 2, 'N': 1})\n", "df['CentralAir_num'] = df.CentralAir.replace({'Y': 1, 'N': 0})\n", "df['Fence_num'] = df.CentralAir.replace({'GdPrv': 2, 'GdWo': 2, 'MnPrv': 1, 'MnWw': 1, 'NoFence': 0})\n", "df['Utilities_num'] = df.Utilities.replace({'AllPub': 1, 'NoSewr': 0, 'NoSeWa': 0, 'ELO': 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2c69771c-c568-78dd-2fec-b09136d81ee1", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []\n", "toDrop_feats = toDrop_feats + ordinal_obj_feats" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "35927e70-dd95-aa59-2644-9b41db461879" }, "outputs": [], "source": [ "df.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "50531f64-fe9e-a407-e161-d080a132d19d" }, "source": [ "**>>>>>>>>>>>>>>>>End of transformation of ordinal categorical features into numerical>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "23f7d81b-c19a-1839-6d85-16c855ff027d" }, "source": [ "**>>>>>>>>>>>>>>>>Some arguable feature engineering>>>>>>>>>>>>>>>>>>**\n", "\n", "**For any cells in this part, if you don't want it, just mark or delete the whole well, it won't affect cells after.**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f3cccfbc-f8fb-9079-10c6-531d42b4a3b0" }, "source": [ "### 1. Try to make quality features and some categorical features more \"significant\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b0f8833e-2cc8-6b97-3dcb-50b82295f1d1", "collapsed": true }, "outputs": [], "source": [ "toDrop_feats = []" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "23b7f84a-1653-5b66-00c4-b236c8f59a98", "collapsed": true }, "outputs": [], "source": [ "# some maybe newer\n", "df['newer_dwelling'] = df.MSSubClass.replace({20: 1, 30: 0, 40: 0, 45: 0, 50: 0, 60: 1, 70: 0, 75: 0,\\\n", " 80: 0, 85: 0, 90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})\n", "\n", "# but we still keep the original feature and transform it into categorical\n", "map_MSS = {x: 'Subclass_'+str(x) for x in df.MSSubClass.unique()}\n", "df['MSSubClass'] = df.MSSubClass.replace(map_MSS)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bccf37d7-4740-0748-0de5-95a6980b3018" }, "outputs": [], "source": [ "# it maybe helpful to transform some quality style features into binary\n", "quality_feats = ['OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num',\\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num']\n", "toDrop_feats = toDrop_feats + quality_feats\n", "\n", "all_data = df\n", "overall_poor_qu = all_data.OverallQual.copy()\n", "overall_poor_qu = 5 - overall_poor_qu\n", "overall_poor_qu[overall_poor_qu<0] = 0\n", "overall_poor_qu.name = 'overall_poor_qu'\n", "\n", "overall_good_qu = all_data.OverallQual.copy()\n", "overall_good_qu = overall_good_qu - 5\n", "overall_good_qu[overall_good_qu<0] = 0\n", "overall_good_qu.name = 'overall_good_qu'\n", "\n", "overall_poor_cond = all_data.OverallCond.copy()\n", "overall_poor_cond = 5 - overall_poor_cond\n", "overall_poor_cond[overall_poor_cond<0] = 0\n", "overall_poor_cond.name = 'overall_poor_cond'\n", "\n", "overall_good_cond = all_data.OverallCond.copy()\n", "overall_good_cond = overall_good_cond - 5\n", "overall_good_cond[overall_good_cond<0] = 0\n", "overall_good_cond.name = 'overall_good_cond'\n", "\n", "exter_poor_qu = all_data.ExterQual_num.copy()\n", "exter_poor_qu[exter_poor_qu<3] = 1\n", "exter_poor_qu[exter_poor_qu>=3] = 0\n", "exter_poor_qu.name = 'exter_poor_qu'\n", "\n", "exter_good_qu = all_data.ExterQual_num.copy()\n", "exter_good_qu[exter_good_qu<=3] = 0\n", "exter_good_qu[exter_good_qu>3] = 1\n", "exter_good_qu.name = 'exter_good_qu'\n", "\n", "exter_poor_cond = all_data.ExterCond_num.copy()\n", "exter_poor_cond[exter_poor_cond<3] = 1\n", "exter_poor_cond[exter_poor_cond>=3] = 0\n", "exter_poor_cond.name = 'exter_poor_cond'\n", "\n", "exter_good_cond = all_data.ExterCond_num.copy()\n", "exter_good_cond[exter_good_cond<=3] = 0\n", "exter_good_cond[exter_good_cond>3] = 1\n", "exter_good_cond.name = 'exter_good_cond'\n", "\n", "bsmt_poor_cond = all_data.BsmtCond_num.copy()\n", "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n", "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n", "bsmt_poor_cond.name = 'bsmt_poor_cond'\n", "\n", "bsmt_good_cond = all_data.BsmtCond_num.copy()\n", "bsmt_good_cond[bsmt_good_cond<=3] = 0\n", "bsmt_good_cond[bsmt_good_cond>3] = 1\n", "bsmt_good_cond.name = 'bsmt_good_cond'\n", "\n", "garage_poor_qu = all_data.GarageQual_num.copy()\n", "garage_poor_qu[garage_poor_qu<3] = 1\n", "garage_poor_qu[garage_poor_qu>=3] = 0\n", "garage_poor_qu.name = 'garage_poor_qu'\n", "\n", "garage_good_qu = all_data.GarageQual_num.copy()\n", "garage_good_qu[garage_good_qu<=3] = 0\n", "garage_good_qu[garage_good_qu>3] = 1\n", "garage_good_qu.name = 'garage_good_qu'\n", "\n", "garage_poor_cond = all_data.GarageCond_num.copy()\n", "garage_poor_cond[garage_poor_cond<3] = 1\n", "garage_poor_cond[garage_poor_cond>=3] = 0\n", "garage_poor_cond.name = 'garage_poor_cond'\n", "\n", "garage_good_cond = all_data.GarageCond_num.copy()\n", "garage_good_cond[garage_good_cond<=3] = 0\n", "garage_good_cond[garage_good_cond>3] = 1\n", "garage_good_cond.name = 'garage_good_cond'\n", "\n", "kitchen_poor_qu = all_data.KitchenQual_num.copy()\n", "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n", "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n", "kitchen_poor_qu.name = 'kitchen_poor_qu'\n", "\n", "kitchen_good_qu = all_data.KitchenQual_num.copy()\n", "kitchen_good_qu[kitchen_good_qu<=3] = 0\n", "kitchen_good_qu[kitchen_good_qu>3] = 1\n", "kitchen_good_qu.name = 'kitchen_good_qu'\n", "\n", "df_qual = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n", " exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n", " garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)\n", "df = pd.concat((df, df_qual), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1154f3e1-a56e-2cfe-2f53-4da2b2c414a5", "collapsed": true }, "outputs": [], "source": [ "# for some categorical features, certain levels may imply better quality\n", "toDrop_feats = toDrop_feats + ['MasVnrType', 'SaleCondition', 'Neighborhood']\n", "\n", "map_Mas = {'BrkCmn': 1, 'BrkFace': 1, 'CBlock': 1, 'Stone': 1, 'None': 0}\n", "MasVnrType_Any = all_data.MasVnrType.replace(map_Mas)\n", "MasVnrType_Any.name = 'MasVnrType_Any'\n", "\n", "map_Sale = {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0}\n", "SaleCondition_PriceDown = all_data.SaleCondition.replace(map_Sale)\n", "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown'\n", "\n", "neigh_good_feats = ['NridgHt', 'Crawfor', 'StoneBr', 'Somerst', 'NoRidge']\n", "df['Neighborhood_good'] = 0\n", "df.loc[df.Neighborhood.isin(neigh_good_feats), 'Neighborhood_good'] = 1\n", "\n", "df = pd.concat((df, MasVnrType_Any, SaleCondition_PriceDown), axis=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "304adfa7-7efa-adca-8910-b1b95e10eb02", "collapsed": true }, "outputs": [], "source": [ "# Monthes with the lagest number of deals may be significant\n", "df['season'] = df.MoSold.replace({1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d487154e-c95b-8059-aa76-2f8df904c0c4" }, "outputs": [], "source": [ "# Numer month is not significant, it maybe helpful to transform them into object feature\n", "map_Mo = {1: 'Yan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Avg', 9: 'Sep', 10: 'Oct', \\\n", " 11: 'Nov', 12: 'Dec'}\n", "df = df.replace({'MoSold': map_Mo})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "54016eac-ee8c-c647-9188-a92dcebeb46e", "collapsed": true }, "outputs": [], "source": [ "# some features about years\n", "# sold at the same year as bulit\n", "df['SoldImmediate'] = 0\n", "df.loc[(df.YrSold == df.YearBuilt), 'SoldImmediate'] = 1\n", "\n", "# reconstructed since first built\n", "df['Recon'] = 0\n", "df.loc[(df.YearBuilt < df.YearRemodAdd), 'Recon'] = 1\n", "\n", "# reconstructed after sold\n", "df['ReconAfterSold'] = 0\n", "df.loc[(df.YrSold < df.YearRemodAdd), 'ReconAfterSold'] = 1\n", "\n", "# reconstructed the same year as sold\n", "df['ReconEqualSold'] = 0\n", "df.loc[(df.YrSold == df.YearRemodAdd), 'ReconEqualSold'] = 1" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6bab8cde-94a1-0d6b-b041-a9f56f13814d", "collapsed": true }, "outputs": [], "source": [ "# Years are too much, it maybe helpful to devide them into groups, and delete the original ones\n", "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n", "df.YearBuilt = df.YearBuilt.map(year_map)\n", "df.YearRemodAdd = df.YearRemodAdd.map(year_map)\n", "df.GarageYrBlt = df.GarageYrBlt.map(year_map)\n", "df.loc[df.GarageYrBlt==0, 'GarageYrBlt'] = 'NoGarage'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7fce71fb-f554-fe73-919b-d5d916d64188" }, "outputs": [], "source": [ "print(len(toDrop_feats), df.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0807997b-8d58-ce43-e33c-4f73c6847fee", "collapsed": true }, "outputs": [], "source": [ "t = ['PavedDrive', 'Functional', 'CentralAir', 'Fence', 'OverallQual', 'OverallCond', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'MasVnrType', 'SaleCondition', 'Neighborhood']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "902e80a3-506f-55d6-ae5b-9eb57928eeec" }, "outputs": [], "source": [ "[x for x in t if x not in toDrop_feats]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "24eebd49-e667-151e-8a8f-9f255f0d54ae", "collapsed": true }, "outputs": [], "source": [ "temp = ['MSSubClass', 'MSZoning', 'LotFrontage', 'LotArea', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'Foundation', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating', 'CentralAir', 'Electrical', 'X1stFlrSF', 'X2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Functional', 'Fireplaces', 'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageCars', 'GarageArea', 'PavedDrive', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', 'X3SsnPorch', 'ScreenPorch', 'PoolArea', 'Fence', 'MiscFeature', 'MiscVal', 'MoSold', 'YrSold', 'SaleType', 'SaleCondition', 'ExterQual_num', 'ExterCond_num', 'BsmtQual_num', 'BsmtCond_num', 'HeatingQC_num', 'KitchenQual_num', 'FireplaceQu_num', 'GarageQual_num', 'GarageCond_num', 'PoolQC_num', 'Functional_num', 'PavedDrive_num', 'CentralAir_num', 'Fence_num', 'Utilities_num', 'newer_dwelling', 'overall_poor_qu', 'overall_good_qu', 'overall_poor_cond', 'overall_good_cond', 'exter_poor_qu', 'exter_good_qu', 'exter_poor_cond', 'exter_good_cond', 'bsmt_poor_cond', 'bsmt_good_cond', 'garage_poor_qu', 'garage_good_qu', 'garage_poor_cond', 'garage_good_cond', 'kitchen_poor_qu', 'kitchen_good_qu', 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Neighborhood_good', 'season', 'SoldImmediate', 'Recon', 'ReconAfterSold', 'ReconEqualSold']" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "acd12611-c90c-44dc-1d3d-d09ddbfd8e63" }, "source": [ "### 2. Do SVC for some features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7b693081-20e3-2097-1889-7182664dc27e" }, "outputs": [], "source": [ "from sklearn.svm import SVC\n", "svm = SVC(C=100)\n", "# price categories\n", "pc = pd.Series(np.zeros(train.shape[0]))\n", "pc[:] = 'pc1'\n", "pc[train.SalePrice >= 150000] = 'pc2'\n", "pc[train.SalePrice >= 220000] = 'pc3'\n", "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n", "X_t = pd.get_dummies(train.loc[:, columns_for_pc])\n", "svm.fit(X_t, pc)\n", "pc_pred = svm.predict(X_t)\n", "\n", "price_category = pd.DataFrame(np.zeros((df.shape[0],1)), columns=['pc'], index=df.index)\n", "X_t = pd.get_dummies(df.loc[:, columns_for_pc])\n", "pc_pred = svm.predict(X_t)\n", "price_category[pc_pred=='pc2'] = 1\n", "price_category[pc_pred=='pc3'] = 2\n", "\n", "toDrop_feats = toDrop_feats + columns_for_pc\n", "df = pd.concat((df, price_category), axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8a2a8a9c-c47c-d444-0d86-61f9f8c4f6b0" }, "source": [ "### 3. Scale the numerical data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "32a19b73-006a-7ad0-b344-5d0f4420b30a", "collapsed": true }, "outputs": [], "source": [ "numeric_feats = df.dtypes[df.dtypes != \"object\"].index\n", "t = df[numeric_feats].quantile(.95)\n", "use_max_scater = t[t == 0].index\n", "use_95_scater = t[t != 0].index\n", "df[use_max_scater] = df[use_max_scater]/df[use_max_scater].max()\n", "df[use_95_scater] = df[use_95_scater]/df[use_95_scater].quantile(.95)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c05ec496-b78c-c430-29fa-bbb5a29ecbc9" }, "source": [ "### 4. Log skewed data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e288856f-c3f2-1dc2-810d-db54685759a5", "collapsed": true }, "outputs": [], "source": [ "df_new = df.copy()\n", "from scipy.stats import skew\n", "numeric_feats = list(df_new.dtypes[df_new.dtypes != \"object\"].index)\n", "feat_skewness = df_new[numeric_feats].apply(lambda x: skew(x.dropna()))\n", "skewed_feats = list(feat_skewness[feat_skewness > 0.75].index)\n", "df_new[skewed_feats] = np.log1p(df_new[skewed_feats])\n", "\n", "df = df_new.copy()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "cf63419a-de14-3a5d-ce32-4126266f898b" }, "source": [ "### 5. Drop some features that have been used to create new features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6fa24eae-6266-4559-670f-ecc09b130939" }, "outputs": [], "source": [ "# print(len(toDrop_feats), toDrop_feats)\n", "# df.drop(toDrop_feats, inplace=True, axis=1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b573f011-7a1c-4f93-040e-ed1b41cfa71d" }, "source": [ "### 6. Get dummy features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cd1d1d0c-1299-eba5-82d1-c53479bd1923" }, "outputs": [], "source": [ "df_new = pd.get_dummies(df)\n", "print(df.shape, df_new.shape)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b2c67eb6-2f30-85a3-6315-770d749cc532" }, "source": [ "### 7. Create interact features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "542bdc17-1240-6deb-30aa-2c8db5cea8f6" }, "outputs": [], "source": [ "from itertools import product, chain\n", "\n", "def poly(X):\n", " areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']\n", " t = chain(df_qual.axes[1].get_values(), \n", " ['OverallQual_num', 'OverallCond_num', 'ExterQual_num', 'ExterCond_num', 'BsmtCond_num', \\\n", " 'GarageQual_num', 'GarageCond_num', 'KitchenQual_num', 'HeatingQC_num', \\\n", " 'MasVnrType_Any', 'SaleCondition_PriceDown', 'Recon',\n", " 'ReconAfterSold', 'SoldImmediate'])\n", " for a, t in product(areas, t):\n", " x = X.loc[:, [a, t]].prod(1)\n", " x.name = a + '_' + t\n", " yield x\n", "\n", "XP = pd.concat(poly(df_new), axis=1)\n", "df_new = pd.concat((df_new, XP), axis=1)\n", "\n", "df_new.shape" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "1919d1c5-5410-8f10-62cc-1cd93e1a0e2d" }, "source": [ "**>>>>>>>>>>>>>>>>End of arguable feature engineering>>>>>>>>>>>>>>>>>>**" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "72885f43-2cb4-b41a-ed26-623a126f1883" }, "source": [ "**>>>>>>>>>>>>>>>>A LASSO cross validation>>>>>>>>>>>>>>>>>>**\n", "\n", "### 1. Cross valication" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "65880a72-7b73-345b-8759-7b970e2632d9" }, "outputs": [], "source": [ "df_final = df_new.copy()\n", "\n", "X_train = df_final[:train.shape[0]]\n", "print(X_train.shape)\n", "y_train = np.log1p(train.SalePrice)\n", "print(y_train.shape)\n", "X_test = df_final[test.shape[0]+1:]\n", "print(X_test.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "93f49499-52b4-1dd2-f0cd-e4ac051f1215", "collapsed": true }, "outputs": [], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "from sklearn.cross_validation import cross_val_score\n", "from sklearn.metrics import make_scorer, mean_squared_error\n", "\n", "def rmse_cv(model, X_train, y_train):\n", " rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring=\"neg_mean_squared_error\", cv=10))\n", " return(rmse)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8d7968f8-edc7-da15-20e3-76fa5e20a526" }, "outputs": [], "source": [ "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result = pd.Series(cv_lasso, index = alphas)\n", "result.plot()\n", "result.min()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a61d42e1-34d5-e558-daba-2dae46632183" }, "source": [ "### 2. Detect outliers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "47f7800d-92b9-9716-8e8f-dfa9ed811cd9", "collapsed": true }, "outputs": [], "source": [ "lasso_test = Lasso(alpha=0.00005, max_iter=1000).fit(X_train, y_train)\n", "y_test = lasso_test.predict(X_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "37c0607a-1901-de5e-902b-feb6c2760a6d" }, "outputs": [], "source": [ "dfy = pd.DataFrame({\"id\": train.Id, \"y_real\": y_train, \"y_test\": y_test, \"residual\": y_test - y_train},\\\n", " columns=['id', 'y_real', 'y_test', 'residual'])\n", "\n", "outliers_id = np.array([524, 1299])\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b08e79bd-58af-8599-d7f3-8c33195e0719" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='y_test', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 2], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bb6e6013-7f5c-1bc3-76a0-338d668b4f64" }, "outputs": [], "source": [ "ax = dfy.plot(x='y_real', y='residual', kind='scatter')\n", "ax.scatter(dfy.iloc[outliers_id, 1], dfy.iloc[outliers_id, 3], c='r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f45b912b-ea94-1055-4f2c-e1cb38599c0c" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "ax = plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "95454d46-b474-2e93-a139-6c5465cb3beb" }, "source": [ "# Two obvious outliers!!!" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ad4aed20-d8bb-6d4e-1f14-161fe65a0e17" }, "outputs": [], "source": [ "outliers_id = np.array([524, 1299])\n", "\n", "outliers_id = outliers_id - 1 # id starts with 1, index starts with 0\n", "X_train = X_train.drop(outliers_id, axis=0)\n", "y_train = y_train.drop(outliers_id, axis=0)\n", "\n", "from sklearn.linear_model import Lasso\n", "#alphas = [0.00001, 0.00002, 0.00003, 0.00004, 0.00005, 0.000075, 0.0001, 0.00025, 0.0005]\n", "alphas = [0.00001, 0.00005, 0.0001, 0.00025, 0.0005, 0.00075, 0.001, 0.0015, 0.002]\n", "cv_lasso = [rmse_cv(Lasso(alpha = alpha, max_iter=100), X_train, y_train).mean() for alpha in alphas];\n", "result2 = pd.Series(cv_lasso, index = alphas)\n", "df_result = pd.concat((result, result2), axis=1)\n", "df_result.columns= ['original', 'without outliers']\n", "df_result.plot()\n", "# result.min()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1541af2e-77f1-7348-0543-b41d04e9d423" }, "outputs": [], "source": [ "# this come from iterational model improvment. I was trying to understand why the model gives to the two points much better price\n", "x_plot = X_train.loc[X_train['SaleCondition_Partial']==1, 'GrLivArea']\n", "y_plot = y_train[X_train['SaleCondition_Partial']==1]\n", "plt.scatter(x_plot, y_plot)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3995ec98-5e2a-a67f-0af5-3a9b4f8b7ec3" }, "outputs": [], "source": [ "lasso = Lasso(alpha=0.00075, max_iter=1000).fit(X_train, y_train)\n", "y_pred = lasso.predict(X_test)\n", "y_pred.shape" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "aa4c697a-37b3-aafd-36d5-6a9a61a05095", "collapsed": true }, "outputs": [], "source": [ "solution = pd.DataFrame({\"id\": test.Id, \"SalePrice\": np.expm1(y_pred)}, columns=['id', 'SalePrice'])\n", "solution.to_csv(\"lasso_sol.csv\", index = False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "35df076a-8a57-7667-50c7-5db93db12fdb" }, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "import seaborn as sns\n", "from scipy.stats import skew\n", "\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ae0c2038-1b72-3f5e-fb08-8b93fba470e6" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\")\n", "test = pd.read_csv(\"../input/test.csv\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "91313991-bfcd-58c8-eabf-78c490cae691" }, "outputs": [], "source": [ "df = pd.concat((train.loc[:, 'MSSubClass':'SaleCondition'], test.loc[:, 'MSSubClass':'SaleCondition']))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0ecad169-2d4a-9604-273b-ee1f5c454e30" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "40f8c131-34f0-da28-f51d-dc8d08c2597f" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2003421d-b7c7-fa12-a61f-a7c4be20b78c" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1d23fc91-ac81-d638-7b5b-9a03dd63a4f5" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "882ca5cb-cbfc-dbe7-afa2-285682c4e296" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bbc47b12-14ce-b59f-a806-bececd267d38" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8ea6833a-984e-d503-25f2-ce9262d4a258" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d95095d1-f552-ca85-99dc-40e5f21598ff" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bfca0506-62a5-7fa5-f1aa-75097124fd55" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3794d049-a7b7-b53b-911e-3a5f9ab654f6" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "94a329f7-ae88-b91b-b670-2c226cf938b2" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c7fe68b7-b27b-328c-3fd0-1822ebf244b8", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0fbd5522-5e53-2be0-d543-c979fcf7e35f" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "621790db-b3a9-ea4f-ebc5-0c7ed19f2344" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9c64e0b0-55f2-f911-0b82-0bd3b20621c0" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4d1b450f-b4b2-5a88-f803-69a636505253", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fa6057a5-781b-3280-cd84-dee2f5f427a2", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2db0ca43-fbd7-7b7b-54e0-b18ea7fb4f40", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "542ac247-10d2-8718-2689-7938f48d3c7a" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a7ec0f79-68f0-f683-225d-b9db47a8bef7" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d9e74e6e-760e-1517-1162-0b133f5fca28" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a8bece03-e652-078f-63cc-4c560149c09a", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "43009583-7d03-27b6-4482-b34bc92421af" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a641aba4-b131-9a1c-af83-384012153d4d", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "61b2b682-3069-049a-585f-626705129818" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "50c6891f-fc50-d494-9b8b-67fcf2b4d78b" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c369763c-2afb-8b30-ab53-c8e5b0b2b003" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "47453c55-8f51-8882-8b7b-748647511d1a", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "90210757-5c72-e100-4abc-f68cdfdc2777" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f348ab45-1679-c2f6-0925-d0b2fd55b2f9" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5de30a34-df54-491f-3caa-3eacff75c31e", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "453999b2-8cbd-73ab-fa9a-ec23dd27844b" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7b5c69eb-70e6-3e38-6ea1-321de088e313" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "687ef28d-cb91-8834-23c1-8152d8f3a264", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "91fa76f7-90ea-7f27-c796-d24efc5832b2" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2a654de9-667d-1f99-fdd4-34e0e2d0da84" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8371548c-d73e-8218-5385-09cd405fd878", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "acb1f9a2-f5ba-94bf-9b68-9b48d8819082", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a1441e78-cf29-eb99-598d-92035a65f7a1" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7aeeaca1-1af5-e5ec-01ea-fc23c467fbaf" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "30cfcdf7-bccc-c7ee-7b23-379be8047785" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5125261b-1d49-e18d-7343-a9d24162ad4e", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "214bff77-9e3d-a714-dcd6-6e660ab66a8e" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "91597bca-7f2f-59dc-c518-bce43b6ff01f", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8c75eee3-6230-89d2-0b99-2e39ef49962a" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "85e2bd1d-e31c-18d7-75a4-439a2a1bc6bb" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8575e20e-b66c-ffb9-8064-121b2c0b746f", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5c132002-9283-7369-95f2-64bbfe4a27be", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "638b444b-31c4-6818-6e5a-f67255079b4f", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "49cfd6c7-0486-a148-b8b1-d26daf31edba" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e057a6a5-5c91-be8a-9c02-c3dca78f8fb1", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9e125e87-0bac-afae-6a6a-4b8137b82950" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "efcd7802-990c-a9bd-38fb-3008923b43bc" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4f7ce28a-d043-24d9-762e-294a2e1a402c" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f2b7fd8f-f24c-bd56-2438-8d0c001ca7f0" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c402242a-4f07-4979-a356-f6bbf68648f1" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5035f880-9824-31f9-bb91-7ca2bbd03af0" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "464df04f-f8e7-c3a8-8a30-950deb7655fe", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "98fadf6b-9cb1-85a8-097c-03cef1952900" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2bba5bc9-acd7-e4dd-313c-a19c8fbebe82" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cc4b6275-0890-5058-8c87-5fc69888330b" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a4c3150f-9af1-c2aa-6f94-9e67072929f7", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5db7f92b-1878-d0d2-79b0-40b5ba864be4", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c4ac0fe4-f0fb-872f-f872-d095eb5d484b", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "09992347-98bb-3f0d-5db3-737d8af15e7f", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a1a198f6-b768-c7f7-76ca-064c0d2be54e", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "836641ba-75ab-5bca-dcac-1b05448386b6", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d73405db-61bf-e744-af8a-d37c2e1f00ac", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bede1531-41d4-c65e-b151-6c37e9cca298", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9cff3556-ed9e-cc74-e9f5-35af19f2c04c", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c678f1f7-bc80-118f-d645-b5d73d8d8bae" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cd5f9734-a134-2523-f730-3d5207daaaa7" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f6110278-82d3-db95-274c-e2b3eb04edec", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f3a40ab9-6608-4548-3417-3fe52182543f" }, "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0f6e3fc7-2e42-e3fb-81a3-82aa10cdf938" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1104d8c0-0bf0-34d7-2c29-5f2fd253af48" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8fbeb000-38bf-9eb5-f95d-28e21631fcca" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e70f6535-e238-b209-1624-9936fab91bc4", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "66a1fb3f-b4f1-642d-6be2-d1d913cabea4", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a2e990fe-152c-9b06-6aea-140fec4c9481" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "55cf6cc6-3f97-c0dc-e82f-6cc5c97fe808" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e5ecb9b9-78c6-8195-313c-da324e4029b1", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9fc99989-ec0f-49f9-bdaf-cf36e75eaba1", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "163240d0-5bc4-9f48-31c2-c1e4e3199fa1", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5b69525b-a344-9971-c840-ce4c4ac32092", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "30eb815d-88e7-396f-85ba-ba6c9c906657", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "324c787b-d596-9824-74f6-842ea1680035" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8e5f0175-541d-badb-3829-a45696aadb67" }, "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "38169b30-63cd-84c7-522c-54b1da80b6ad" }, "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "4ac3b80a-3ade-b8be-2cc7-1263058ade41" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c9116e0-c464-c2b2-8454-b7a95415fd5e", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "edfbb6e0-d281-5a44-77fc-dc0a6780478b", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1c8bb998-b86e-9e0b-14a7-64f1eccaf719" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "274bd0f0-5c9b-ccb4-867b-c44512b6cd20", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d2a801f1-9caf-7572-7009-349902bd5a36", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "023ff440-fa87-b565-8af1-35aaad834ef6" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c1c7bd66-85f6-c9de-9a0f-6ffe30e5d551", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "777fbdbb-4479-44d4-aeb1-c809006215f5", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7d84c4fd-3e7b-75aa-a29d-aa26a818aa51" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "90b2bf5d-d7c4-9f42-46f9-6a005f099583", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "406f0262-2172-35a1-9d17-31ca8c136a51" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5a0ed3b0-40c4-dfb0-c810-e26bfb4b42cf", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "1e226ae0-0a55-b0cb-c753-e4cca02e57c1" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "02211b6d-d917-86b1-3d55-24b17be05355" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8994d473-ad40-c6d1-4695-170073ebb5fc" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "84a7ee09-4b52-278c-a95a-227bf7f2c76f", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "70c70e9f-c4af-4a8a-937a-37ca60b902dd" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "33e6bb5c-9b5f-6510-1d57-ae1bcb8cada9", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "1bd72492-f77b-113b-2827-641d2692c231" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1599aa2e-26ef-a959-2d16-3f7ee2d27380" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "4ce528cf-2583-9479-306f-ccdd8ab53bbd" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "fe6c6b91-7780-ac15-dded-82300725677f" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "fc058bd5-5b9c-fec2-b4d7-a02dada43111" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a730bd14-9e43-33d3-b635-3c3370314f6b" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "72ca2774-a9e9-0e5e-afa8-3ccc1b933c04" }, "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "be44e2fe-a4a7-53cc-daad-edafa3fb5aae" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0a212197-79b0-6d63-c622-2a75e5f55ff6" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "08c077e2-c4f9-9bd4-8067-b8616497f684", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "06865890-6849-f7c9-3a39-a969e5614417" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "c0acf9b8-d0ff-d0ad-893a-7d589df876d2" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "202f45d8-00b5-c9ff-ba11-d632aa362ed5", "collapsed": true }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "67fde1a1-bb40-1399-5359-5b828fe53258" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2c65c09f-71fa-1897-b09c-c1fa50fb1f1c" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3e7fe4ad-480b-8dab-8157-4ed23df78381" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "14cc36b5-d0af-392c-26e1-362ae33a9e6e" }, "outputs": [], "source": "" }, { "cell_type": "markdown", "metadata": { "_cell_guid": "61707976-32fe-1cca-18b5-b3f562e06f88" }, "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "6e9dfa7f-f783-03a4-12a7-e9944244cdf1" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "01f32c70-1e9c-891c-1f73-98ed5bf05eaf" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cfed02a3-afcf-592c-1bf4-120d4603699f" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c7347d6d-480e-ca20-6233-c4266b2743cf", "collapsed": true }, "outputs": [], "source": "" } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }