---
title: 'Data Wrangling for Real Estate'
author: 'Andy Niggles'
date: 'March 9 2017'
output: 
    html_document:
        number_sections: true
        toc: true
        fig_width: 7
        fig_height: 4.5
        theme: readable
        highlight: tango
---


#Data Wrangling
For sake of length, I'll identify major examples of each kind of wrangling to discuss, but won't go into detail on each occurence within the 90+ values.

##Discrete vs. Continuous
First things first, lets understand the value categories a little better. Lets run through each one and determine whether its possible values are a finite set of choices (discrete) or a continuous value. 
```{r}
library(dummies)
#Load Data
test <- read.csv("../input/test.csv")
train <- read.csv("../input/train.csv")
submission <- read.csv("../input/sample_submission.csv")
str(train)
#Anything thats a Factor type is automatically discrete, so that takes care of columns 3,6-17,22-26, 28-34,36,40-43,54,56, 58-59,61,64-66,73-74,76-78,82,83

#Some of the integer columns may actually be discrete as well, lets look closer (or just look at the accompanying text file that describes each column, but that feels like cheating). If there are a small number of unique values, they may function better as discrete factors instead of integers.
unique(train$MSSubClass)  #This is an example of a numeric column better served as a Factor rather than integer, since the numbers are actually codes/classifications. We'll switch this to a Factor type.
train <- transform(train, MSSubClass = as.numeric(MSSubClass))

unique(train$OverallQual) #This has only 10 unique values, but they represent a position on a spectrum/scale, so they should remain integers and be treated similarly to continuous variables.
unique(train$YearBuilt) #All years will be kept as integers since they are positional
unique(train$BsmtFullBath)  #"Feature counts" like number of beds, baths, fireplaces etc will have few unique values, but are still quantities that can be compared and valued, so they'll remain integers

```

##Missing Values
Lets examine the NA values floating around in our data and determine whether they belong or can be replaced

```{r}
sapply(train, function(x) sum(is.na(x)))

summary(train$LotFrontage)
sumFrontage <- sum(train$LotFrontage, na.rm = TRUE)
sumArea <- sum(train[is.na(train$LotFrontage) == FALSE,5])
frontageMultiplier <- sumFrontage/sumArea
train[is.na(train$LotFrontage),"LotFrontage"] <- train[is.na(train$LotFrontage),"LotArea"]*frontageMultiplier

levels(train$Alley)
levels(train$Alley) <- c(levels(train$Alley), "None")
train[is.na(train$Alley),"Alley"] <- "None"

summary(train$MasVnrArea)
summary(train[is.na(train$MasVnrArea),"MasVnrType"])
summary(train$MasVnrType)
train[is.na(train$MasVnrArea),"MasVnrArea"] <- 0
train[is.na(train$MasVnrType),"MasVnrType"] <- "None"

levels(train$BsmtQual)
head(train[is.na(train$BsmtQual),30:40])
summary(train$TotalBsmtSF)

summary(train$Electrical)
train[is.na(train$Electrical),"Electrical"] <- "SBrkr"

summary(train$FireplaceQu)

summary(train$GarageType)
train[is.na(train$GarageYrBlt),"GarageYrBlt"] <- train[is.na(train$GarageYrBlt),"YearBuilt"] 

summary(train$PoolQC)
summary(train$PoolArea)
train$PoolQC <- NULL

summary(train$Fence)

summary(train$MiscFeature)

```

##Factors to Booleans
As we saw above, many of the factor variables each have a large handful of levels, which (if kept unchanged) will each be converted into a dummy variable, which will really gum up our computation speeds. Lets see what we can consolidate, then we'll convert these factors to booleans using the Dummies package.

```{r}
str(train)
summary(train)

train$MSSubClass <- train$MSSubClass
summary(train$MSSubClass) #Looking at the description file we can see that these codes explain
#levels(train$MSSubClass) <-  c(levels(train$MSSubClass), "Standard","PUD")
train[train$MSSubClass < 120,"MSSubClass"] <- 0
train[train$MSSubClass >= 120,"MSSubClass"] <- 1

summary(train$MSZoning)
levels(train$MSZoning) <- c(levels(train$MSZoning),"Other")
train[train$MSZoning %in% c("C (all)", "FV","RH"),"MSZoning"] <- "Other"

summary(train$LotShape)
levels(train$LotShape) <- c(levels(train$LotShape),"Irr")
train[train$LotShape %in% c("IR1", "IR2","IR3"),"LotShape"] <- "Irr"

summary(train$LandContour)
levels(train$LandContour) <- c(levels(train$LandContour),"UnLvl")
train[train$LandContour %in% c("Bnk", "HLS","Low"),"LandContour"] <- "UnLvl"

train$Utilities <- NULL

summary(train$LotConfig)
train[train$LotConfig %in% c("FR2", "FR3"),"LotConfig"] <- "Corner"

summary(train$Neighborhood)

summary(train$Condition2)

summary(train$HouseStyle)
train[train$HouseStyle %in% c("1.5Fin", "1.5Unf"),"HouseStyle"] <- "1Story"
train[train$HouseStyle %in% c("2.5Fin", "2.5Unf"),"HouseStyle"] <- "2Story"

summary(train$RoofStyle)
levels(train$RoofStyle) <- c(levels(train$RoofStyle),"Other")
train[train$RoofStyle %in% c("Flat", "Shed","Gambrel","Mansard"),"RoofStyle"] <- "Other"

summary(train$RoofMatl)
levels(train$RoofMatl) <- c(levels(train$RoofMatl),"Other")
train[train$RoofMatl %in% c("ClyTile", "Membran","Metal","Roll","Tar&Grv","WdShake","WdShngl"),"RoofMatl"] <- "Other"

summary(train$Exterior1st)
levels(train$Exterior1st) <- c(levels(train$Exterior1st),"Other")
train[train$Exterior1st %in% c("AsbShng", "AsphShn","BrkComm","CBlock","ImStucc","Stone","Stucco","WdShing"),"Exterior1st"] <- "Other"

summary(train$Exterior2nd)
train[train$Exterior2nd %in% c("AsbShng", "AsphShn","BrkComm","CBlock","ImStucc","Stone","Stucco","WdShing"),"Exterior2nd"] <- "Other"

summary(train$MasVnrType)
levels(train$MasVnrType) <- c(levels(train$MasVnrType),"Brick")
train[train$MasVnrType %in% c("BrkCmn", "BrkFace"),"MasVnrType"] <- "Brick"

summary(train$ExterQual)
levels(train$ExterQual) <- c(levels(train$ExterQual),"Average","AboveAverage","BelowAverage")
train[train$ExterQual %in% c("Ex", "Gd"),"ExterQual"] <- "AboveAverage"
train[train$ExterQual %in% c("TA"),"ExterQual"] <- "Average"
train[train$ExterQual %in% c("Po", "Fa"),"ExterQual"] <- "BelowAverage"

summary(train$ExterCond)
levels(train$ExterCond) <- c(levels(train$ExterCond),"Average","AboveAverage","BelowAverage")
train[train$ExterCond %in% c("Ex", "Gd"),"ExterCond"] <- "AboveAverage"
train[train$ExterCond %in% c("TA"),"ExterCond"] <- "Average"
train[train$ExterCond %in% c("Po", "Fa"),"ExterCond"] <- "BelowAverage"

summary(train$Electrical)
levels(train$Electrical) <- c(levels(train$Electrical),"Other")
train[train$Electrical %in% c("FuseF", "FuseP","Mix"),"Electrical"] <- "Other"

summary(train$Foundation)
levels(train$Foundation) <- c(levels(train$Foundation),"Other")
train[train$Foundation %in% c("Slab", "Stone","Wood"),"Foundation"] <- "Other"

summary(train$BsmtQual)
levels(train$BsmtQual) <- c(levels(train$BsmtQual),"Average","AboveAverage","BelowAverage")
train[train$BsmtQual %in% c("Ex", "Gd"),"BsmtQual"] <- "AboveAverage"
train[train$BsmtQual %in% c("TA"),"BsmtQual"] <- "Average"
train[train$BsmtQual %in% c("Po", "Fa"),"BsmtQual"] <- "BelowAverage"

summary(train$BsmtCond)
levels(train$BsmtCond) <- c(levels(train$BsmtCond),"Average","AboveAverage","BelowAverage")
train[train$BsmtCond %in% c("Ex", "Gd"),"BsmtCond"] <- "AboveAverage"
train[train$BsmtCond %in% c("TA"),"BsmtCond"] <- "Average"
train[train$BsmtCond %in% c("Po", "Fa"),"BsmtCond"] <- "BelowAverage"

summary(train$Heating)
levels(train$Heating) <- c(levels(train$Heating),"Other")
train[train$Heating %in% c("Floor", "GasW","Grav","OthW","Wall"),"Heating"] <- "Other"

summary(train$HeatingQC)
levels(train$HeatingQC) <- c(levels(train$HeatingQC),"Average","AboveAverage","BelowAverage")
train[train$HeatingQC %in% c("Ex", "Gd"),"HeatingQC"] <- "AboveAverage"
train[train$HeatingQC %in% c("TA"),"HeatingQC"] <- "Average"
train[train$HeatingQC %in% c("Po", "Fa"),"HeatingQC"] <- "BelowAverage"

summary(train$Functional)
levels(train$Functional) <- c(levels(train$Functional),"Major","Minor","Damaged")
train[train$Functional %in% c("Maj1", "Maj2"),"Functional"] <- "Major"
train[train$Functional %in% c("Min1","Min2"),"Functional"] <- "Minor"
train[train$Functional %in% c("Sev", "Mod"),"Functional"] <- "Damaged"

summary(train$FireplaceQu)
levels(train$FireplaceQu) <- c(levels(train$FireplaceQu),"Average","AboveAverage","BelowAverage")
train[train$FireplaceQu %in% c("Ex", "Gd"),"FireplaceQu"] <- "AboveAverage"
train[train$FireplaceQu %in% c("TA"),"FireplaceQu"] <- "Average"
train[train$FireplaceQu %in% c("Po", "Fa"),"FireplaceQu"] <- "BelowAverage"

summary(train$GarageType)
train[train$GarageType %in% c("2Types", "Basment","BuiltIn"),"GarageType"] <- "Attchd"
train[train$GarageType %in% c("CarPort"),"GarageType"] <- NA

summary(train$GarageQual)
levels(train$GarageQual) <- c(levels(train$GarageQual),"Average","AboveAverage","BelowAverage")
train[train$GarageQual %in% c("Ex", "Gd"),"GarageQual"] <- "AboveAverage"
train[train$GarageQual %in% c("TA"),"GarageQual"] <- "Average"
train[train$GarageQual %in% c("Po", "Fa"),"GarageQual"] <- "BelowAverage"

summary(train$GarageCond)
levels(train$GarageCond) <- c(levels(train$GarageCond),"Average","AboveAverage","BelowAverage")
train[train$GarageCond %in% c("Ex", "Gd"),"GarageCond"] <- "AboveAverage"
train[train$GarageCond %in% c("TA"),"GarageCond"] <- "Average"
train[train$GarageCond %in% c("Po", "Fa"),"GarageCond"] <- "BelowAverage"

summary(train$Fence)
levels(train$Fence) <- c(levels(train$Fence), "None","Fence")
train[is.na(train$Fence) == FALSE,"Fence"] <- "Fence"

summary(train$MiscFeature)
train[train$MiscFeature %in% c("Gar2", "Othr","TenC"),"MiscFeature"] <- NA

summary(train$SaleType)
train[train$SaleType %in% c("Con","ConLD","ConLI","CWD","ConLw", "Fa"),"SaleType"] <- "Oth"

summary(train$SaleCondition)
train[train$SaleCondition %in% c("AdjLand","Alloca","Family",NA),"SaleCondition"] <- "Abnorml"

train <- droplevels(train)


trainEX <- dummy.data.frame(train)
#dim(trainEX)
#trainEX
#AlleyNone
#LotShapeIrr
#LandContourLvl
```