--- title: "Exploratory Analysis with Commentary" author: "Michael Stone" date: "12 February 2017" output: html_document --- # Loading Packages ```{r setup, warning=FALSE} # setup, load packages & data library(jsonlite) library(dplyr) library(tidyr) library(ggplot2) library(maps) library(mapproj) library(RColorBrewer) library(wordcloud) ``` # Loading training data ```{r, warning=FALSE} KAGGLE <- TRUE train_path <- ifelse(KAGGLE, "../input/train.json", "data/train.json") test_path <- ifelse(KAGGLE, "../input/test.json", "data/test.json") train.json <- jsonlite::fromJSON(train_path, flatten = TRUE) test.json <- jsonlite::fromJSON(test_path, flatten = TRUE) ``` ```{r, warning=FALSE} # Select just the features I need for now & bind into a dataframe train.json <- train.json[c("created", "bathrooms", "bedrooms", "price", "latitude", "longitude", "building_id", "manager_id", "features", "interest_level")] train.df <- as.data.frame(bind_cols(train.json)) train.df$created <- as.Date(unlist(train.df$created)) train.df$bathrooms <- as.numeric(unlist(train.df$bathrooms)) train.df$bedrooms <- as.numeric(unlist(train.df$bedrooms)) train.df$price <- as.numeric(unlist(train.df$price)) train.df$longitude <- as.numeric(unlist(train.df$longitude)) train.df$latitude <- as.numeric(unlist(train.df$latitude)) train.df$building_id <- as.character(unlist(train.df$building_id)) train.df$manager_id <- as.character(unlist(train.df$manager_id)) train.df$interest_level <- as.factor(unlist(train.df$interest_level)) train.df$interest_level <- factor(train.df$interest_level, levels = c("high", "medium", "low")) ``` ```{r, warning=FALSE} head(train.df, n = 1) ``` ## interest summary ```{r, warning=FALSE} summary(train.df$interest_level) ``` ## price and interest level ```{r, warning=FALSE} ggplot(train.df) + stat_boxplot(aes(interest_level, price)) ``` ## Plot distance density by interest_level ```{r, warning=FALSE} # New York City Center Coords ny_lat <- 40.785091 ny_lon <- -73.968285 # Alternate New York City Center Coords #ny_center <- geocode("new york", source = "google") # Add Euclidean Distance to City Center train.df$distance_city <- mapply(function(lon, lat) sqrt((lon - ny_lon)^2 + (lat - ny_lat)^2), train.df$longitude, train.df$latitude) # Discard real state far from City Center ny_outliners_dist <- 0.2 ggplot(train.df[train.df$distance_city < ny_outliners_dist, ], aes(distance_city, color = interest_level)) + geom_density() ``` ## NY Rental Map ```{r, warning=FALSE} g <- ggplot(train.df, aes(x=longitude, y=latitude)) g <- g + geom_point(aes(color = interest_level), size = 1) g <- g + borders("county") g <- g + coord_map("ortho", orientation = c(40.7, -73.95, 0), xlim = c(-74.15, -73.75), ylim = c(40.55, 40.9)) g <- g + ggtitle("Interest Level by Location") g ``` ```{r, warning=FALSE} g <- ggplot(train.df, aes(x=longitude, y=latitude)) g <- g + geom_point(aes(color = interest_level), size = 1) g <- g + borders("county") g <- g + coord_map("ortho", orientation = c(40.75, -74, 0), xlim = c(-74.02, -73.95), ylim = c(40.72, 40.79)) g <- g + ggtitle("Interest Level by Location - Manhattan") g ``` ## Most Popular Property Managers ```{r, warning=FALSE} manager.df <- train.df %>% group_by(manager_id, interest_level) %>% summarise(no_rows = length(interest_level)) %>% spread(interest_level, no_rows) %>% filter(!is.na(high)) manager.df$medium[is.na(manager.df$medium)] <- 0 manager.df$low[is.na(manager.df$low)] <- 0 manager.df <- filter(manager.df, (low + medium + high) > 20) manager.df <- manager.df %>% mutate(per = 100 * high / (low + medium + high)) manager.df <- arrange(manager.df, desc(per)) manager.df <- head(manager.df, 15) g <- ggplot(manager.df, aes(x = reorder(manager_id, per, sum), y = per)) g <- g + labs(x="Manager Id", y="High Interest (% of Total Listings)") g <- g + ggtitle("Most Popular Managers") + theme(plot.title = element_text(hjust = 0.5)) g <- g + geom_bar(stat = "identity", colour = "green", fill = "lightgreen") + coord_flip() g <- g + theme(legend.position="bottom", legend.direction="horizontal", legend.title = element_blank()) g ``` ## Most Popular Buildings ```{r, warning=FALSE} building.df <- train.df %>% filter(building_id != 0) %>% group_by(building_id, interest_level) %>% summarise(no_rows = length(interest_level)) %>% spread(interest_level, no_rows) %>% filter(!is.na(high)) building.df$medium[is.na(building.df$medium)] <- 0 building.df$low[is.na(building.df$low)] <- 0 building.df <- filter(building.df, (low + medium + high) > 10) building.df <- building.df %>% mutate(per = 100 * high / (low + medium + high)) building.df <- arrange(building.df, desc(per)) building.df <- head(building.df, 15) g <- ggplot(building.df, aes(x = reorder(building_id, per, sum), y = per)) g <- g + labs(x="Building Id", y="High Interest (% of Total Listings)") g <- g + ggtitle("Most Popular Buildings") + theme(plot.title = element_text(hjust = 0.5)) g <- g + geom_bar(stat = "identity", colour = "green", fill = "lightgreen") + coord_flip() g <- g + scale_fill_brewer(palette = "Blues") g ``` ```{r, warning=FALSE} building.loc.df <- filter(train.df, building_id %in% building.df$building_id) g <- ggplot(building.loc.df, aes(x=longitude, y=latitude)) g <- g + geom_point(aes(color = building_id), size = 1.5) g <- g + scale_fill_gradientn(colours=topo.colors(15)) g <- g + borders("county") g <- g + coord_map("ortho", orientation = c(40.7, -73.95, 0), xlim = c(-74.05, -73.8), ylim = c(40.65, 40.85)) g <- g + ggtitle("Most Popular Buildings - Location") g ``` ## Time Series Analysis ```{r, warning=FALSE} time.df <- train.df %>% group_by(created) %>% count(interest_level) g <- ggplot(time.df, aes(created, n)) g <- g + geom_line(aes(color = interest_level), size = 0.75) g <- g + ylab("Number of Listings") + theme(legend.position = "bottom") g ``` ```{r, warning=FALSE} g <- ggplot(time.df, aes(created, fill = interest_level)) g <- g + geom_bar(aes(weight = n), position = "fill") g <- g + ylab("Proportion of Listings") + theme(legend.position = "bottom") g ``` ## Most Popular Property Features ```{r, warning=FALSE} features.high <- train.df %>% filter(interest_level == "high") %>% select(features) features.medium <- train.df %>% filter(interest_level == "medium") %>% select(features) features.low <- train.df %>% filter(interest_level == "low") %>% select(features) features.high <- data.frame(features = unlist(features.high)) %>% group_by(features) %>% summarise(feature_count = n()) %>% arrange(desc(feature_count)) %>% slice(1:40) features.medium <- data.frame(features = unlist(features.medium)) %>% group_by(features) %>% summarise(feature_count = n()) %>% arrange(desc(feature_count)) %>% slice(1:40) features.low <- data.frame(features = unlist(features.low)) %>% group_by(features) %>% summarise(feature_count = n()) %>% arrange(desc(feature_count)) %>% slice(1:40) ``` #### High Interest: ```{r} wordcloud(features.high$features, features.high$feature_count, scale=c(2.8,.4), colors = "red") ``` #### Medium Interest: ```{r} wordcloud(features.medium$features, features.medium$feature_count, scale=c(2.8,.4), colors = "green") ``` #### Low Interest: ```{r} wordcloud(features.low$features, features.low$feature_count, scale=c(2.8,.4), colors = "blue") ```