{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Stock Kaggle Code\nThis will get added to any R notebook","metadata":{}},{"cell_type":"code","source":"# This R environment comes with many helpful analytics packages installed\n# It is defined by the kaggle/rstats Docker image: https://github.com/kaggle/docker-rstats\n# For example, here's a helpful package to load\n\nlibrary(tidyverse) # metapackage of all tidyverse packages\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nlist.files(path = \"../input\")\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-07-11T21:28:16.959260Z","iopub.execute_input":"2022-07-11T21:28:16.962652Z","iopub.status.idle":"2022-07-11T21:28:18.748151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data\n## Let's create some path constants and load up our training data set","metadata":{}},{"cell_type":"code","source":"# Set R not to show scientific notation\noptions(scipen=999)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:36:33.167980Z","iopub.execute_input":"2022-07-11T21:36:33.199802Z","iopub.status.idle":"2022-07-11T21:36:33.212806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PROJECT <- 'house-prices-advanced-regression-techniques'\n# file.path joins path elements in a Operating System agnostic way\n#   i.e. this will work on linux, windows, mac, etc\nDATA_PATH <- file.path(\"..\", \"input\", PROJECT, \"train.csv\")\ntrain <- read_csv(DATA_PATH)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:36:40.977464Z","iopub.execute_input":"2022-07-11T21:36:40.979091Z","iopub.status.idle":"2022-07-11T21:36:41.344930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's look at a small sample of records","metadata":{}},{"cell_type":"code","source":"head(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:36:54.466816Z","iopub.execute_input":"2022-07-11T21:36:54.468473Z","iopub.status.idle":"2022-07-11T21:36:54.510511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## str() is a function that displays the structure of an R object in a relatively compact manner","metadata":{}},{"cell_type":"code","source":"str(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:37:19.462238Z","iopub.execute_input":"2022-07-11T21:37:19.463817Z","iopub.status.idle":"2022-07-11T21:37:19.581060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What kind of object is our train data frame by the way?","metadata":{}},{"cell_type":"code","source":"class(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:37:37.223251Z","iopub.execute_input":"2022-07-11T21:37:37.225047Z","iopub.status.idle":"2022-07-11T21:37:37.240991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How many records and columns does our data have?","metadata":{}},{"cell_type":"code","source":"dim(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:37:59.707823Z","iopub.execute_input":"2022-07-11T21:37:59.709376Z","iopub.status.idle":"2022-07-11T21:37:59.724760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nrow(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:03.474668Z","iopub.execute_input":"2022-07-11T21:38:03.476336Z","iopub.status.idle":"2022-07-11T21:38:03.493144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ncol(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:05.218615Z","iopub.execute_input":"2022-07-11T21:38:05.220163Z","iopub.status.idle":"2022-07-11T21:38:05.236975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Price\n## If we're predicting price, what does it look like?","metadata":{}},{"cell_type":"code","source":"str(train$SalePrice)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:14.732037Z","iopub.execute_input":"2022-07-11T21:38:14.733737Z","iopub.status.idle":"2022-07-11T21:38:14.749185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"range(train$SalePrice)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:26.475530Z","iopub.execute_input":"2022-07-11T21:38:26.477150Z","iopub.status.idle":"2022-07-11T21:38:26.496376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(train$SalePrice)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:35.993423Z","iopub.execute_input":"2022-07-11T21:38:35.995039Z","iopub.status.idle":"2022-07-11T21:38:36.015213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(is.na(train$SalePrice))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:44.296916Z","iopub.execute_input":"2022-07-11T21:38:44.298591Z","iopub.status.idle":"2022-07-11T21:38:44.318298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"any(is.na(train$SalePrice))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:48.377813Z","iopub.execute_input":"2022-07-11T21:38:48.380444Z","iopub.status.idle":"2022-07-11T21:38:48.400035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"require(scales)\n\nqplot(train$SalePrice, geom=\"histogram\", binwidth=5000) + scale_x_continuous(labels = comma)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:38:53.645102Z","iopub.execute_input":"2022-07-11T21:38:53.646682Z","iopub.status.idle":"2022-07-11T21:38:54.283793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qplot(train$SalePrice, geom=\"histogram\", binwidth=100000) + scale_x_continuous(labels = comma)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:39:12.492883Z","iopub.execute_input":"2022-07-11T21:39:12.494654Z","iopub.status.idle":"2022-07-11T21:39:12.772967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Round prices and show frequency instead\nWe could also round the price by doing division and throwing away the remainder and show frequency counts with the `table()` function:","metadata":{}},{"cell_type":"code","source":"bucket_size <- 100000\n\nrounded_prices <- (train$SalePrice %/% bucket_size) * bucket_size\ntable(rounded_prices)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:39:24.051363Z","iopub.execute_input":"2022-07-11T21:39:24.052928Z","iopub.status.idle":"2022-07-11T21:39:24.079896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qplot(train$YearBuilt, geom=\"histogram\", binwidth=1) + scale_x_continuous(labels = comma)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:39:36.531772Z","iopub.execute_input":"2022-07-11T21:39:36.533315Z","iopub.status.idle":"2022-07-11T21:39:36.832164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What's a Numeric Feature You Think Relates to Price?\nSome candidates:\n* YearBuilt\n* YearSold\n* LotArea\n* LotFrontage\n* GrLivArea\n* WoodDeckSF\n* PoolArea","metadata":{}},{"cell_type":"code","source":"cor_matrix <- train %>% \n    select(YearBuilt, LotArea, GrLivArea, SalePrice) %>% \n    cor()\ncor_matrix","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:42:30.287203Z","iopub.execute_input":"2022-07-11T21:42:30.288905Z","iopub.status.idle":"2022-07-11T21:42:30.318045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heatmap(cor_matrix)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:42:38.025995Z","iopub.execute_input":"2022-07-11T21:42:38.027585Z","iopub.status.idle":"2022-07-11T21:42:38.131700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(train %>% \n    select(YearBuilt, LotArea, GrLivArea, SalePrice) %>%\n    is.na())","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:43:36.969981Z","iopub.execute_input":"2022-07-11T21:43:36.972616Z","iopub.status.idle":"2022-07-11T21:43:37.005570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mdl <- glm(SalePrice ~ GrLivArea, data = train)\nmdl","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:44:52.176023Z","iopub.execute_input":"2022-07-11T21:44:52.177766Z","iopub.status.idle":"2022-07-11T21:44:52.209699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(mdl)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:45:13.574579Z","iopub.execute_input":"2022-07-11T21:45:13.576826Z","iopub.status.idle":"2022-07-11T21:45:13.614345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot(mdl)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:45:29.752921Z","iopub.execute_input":"2022-07-11T21:45:29.754545Z","iopub.status.idle":"2022-07-11T21:45:30.401761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qplot(y = SalePrice, x= GrLivArea, data=train) + geom_smooth()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:47:05.544787Z","iopub.execute_input":"2022-07-11T21:47:05.547420Z","iopub.status.idle":"2022-07-11T21:47:07.385157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"?cor","metadata":{"execution":{"iopub.status.busy":"2022-07-11T21:48:17.960348Z","iopub.execute_input":"2022-07-11T21:48:17.962126Z","iopub.status.idle":"2022-07-11T21:48:18.150211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}