{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initial set up","metadata":{}},{"cell_type":"code","source":"install.packages(\"OpenStreetMap\")\nlibrary(tidyverse)\nlibrary(magrittr)\nlibrary(Metrics)\nlibrary(plotly)\nlibrary(OpenStreetMap)\nlibrary(maps)\nlibrary(zeallot)\nlibrary(lubridate)\nlibrary(cowplot)\nlibrary(forecast)\n#packages to be used\nlibrary(lubridate)\nlibrary(corrplot)\nlibrary(scales)\nlibrary(repr)\nlibrary(caTools)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:19:43.746563Z","iopub.execute_input":"2022-07-21T03:19:43.748761Z","iopub.status.idle":"2022-07-21T03:20:57.293195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/store-sales-time-series-forecasting/'\ndf = read_csv(str_c(data_dir,'train.csv'))\ntransactions = read_csv(str_c(data_dir,'transactions.csv'))\nstores = read_csv(str_c(data_dir,'stores.csv'))\noil = read_csv(str_c(data_dir,'oil.csv'))\nhol_events = read_csv(str_c(data_dir,'holidays_events.csv'))\nsub = read_csv(str_c(data_dir,'sample_submission.csv'))\n# test = read_csv(str_c(data_dir,'test.csv'))\n# cities <- read_csv('../input/world-cities/worldcities.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:20:57.295499Z","iopub.execute_input":"2022-07-21T03:20:57.326806Z","iopub.status.idle":"2022-07-21T03:21:01.228949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CLEANING HOLIDAY_EVENTS.csv\n\n#renaming column \nhol_events <- hol_events %>% rename(holiday = type)\n\n#changing holidays that were transferred to 'none', so that they are not correlated as a holiday\nhol_events <- hol_events %>% mutate(holiday = ifelse(transferred == \"True\",\n                        \"None\",\n                        holiday))\n\n#changing 'transfer' to 'holiday' becaues these are the actual days certain holidays were celebrated on\nhol_events <- hol_events %>%\nmutate(holiday = ifelse(holiday == 'Transfer',\n                       'Holiday',\n                       holiday))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:01.231369Z","iopub.execute_input":"2022-07-21T03:21:01.232764Z","iopub.status.idle":"2022-07-21T03:21:01.306117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Joining files\n#joining tables \ndf <- left_join(df, stores, \n             by=\"store_nbr\")\ndf <- left_join(df, oil,\n                 by = 'date')\ndf <- left_join(df, transactions,\n                 by = c('date', 'store_nbr'))\ndf <- left_join(df, hol_events %>%\n                 select(-description, -transferred),\n                 by = c('date'))\n\n#viewing top 5 rows\nhead(df, 5)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:01.308474Z","iopub.execute_input":"2022-07-21T03:21:01.309917Z","iopub.status.idle":"2022-07-21T03:21:06.735908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"#viewing summary statistics\nsummary(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:06.738169Z","iopub.execute_input":"2022-07-21T03:21:06.739499Z","iopub.status.idle":"2022-07-21T03:21:08.332627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#populating missing holiday values with 'none' to indicate it wasnt a holiday\ndf$holiday[is.na(df$holiday)] <- 'None'\n\n#viewing missing values as a list \nas.list(colSums(is.na(df)))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:08.334892Z","iopub.execute_input":"2022-07-21T03:21:08.336242Z","iopub.status.idle":"2022-07-21T03:21:08.676954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#changing to factor type\ndf$store_nbr <- as.factor(df$store_nbr)\n\n#date needs to be date format\ndf$date <- as_date(df$date)\n\n#viewing structure to see results\nstr(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:08.679173Z","iopub.execute_input":"2022-07-21T03:21:08.680520Z","iopub.status.idle":"2022-07-21T03:21:12.331149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding some datetime features","metadata":{}},{"cell_type":"code","source":"library(dplyr)\n#mutate() of dplyr package allows for easy adding and manipulation of columns\ndf <- df %>% mutate(Dayofweek = weekdays(df$date),\n         Monthofyear = months(df$date),\n         Year = year(df$date),\n         YrMonth = format(df$date, \"%Y-%m\"))\n\n#setting months to factors with ordered levels\ndf$Monthofyear <- factor(df$Monthofyear, levels = month.name)\n\n#day of week to factor as well\ndf$Dayofweek <- factor(df$Dayofweek, levels = c('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'))\n\n#viewing sample of 5 random rows\ndf[sample(nrow(df), 5),]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:12.333407Z","iopub.execute_input":"2022-07-21T03:21:12.334765Z","iopub.status.idle":"2022-07-21T03:21:17.554297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#global command makes all plots and graphs larger\noptions(repr.plot.width = 14, repr.plot.height = 8)\n\n#choosing numeric data after removing incomplete observations\nc1 <- cor(df %>% \n            filter(complete.cases(df)) %>%\n            select(sales, onpromotion, dcoilwtico, transactions, Year))\n\n #creating correlation matrix                  \ncorrplot(c1,\n         method = c('color'),\n         addCoef.col = \"black\",\n         addgrid.col = \"black\",\n         tl.col = \"black\",\n        order = 'hclust')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:17.558161Z","iopub.execute_input":"2022-07-21T03:21:17.560583Z","iopub.status.idle":"2022-07-21T03:21:18.496662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#function to create histograms, taking two arguments: x=data and y=title\nhist_function <- function(x, y){\n    x_omit <- na.omit(x)\n    \n    hist(x_omit,\n        main = y,\n        xlab = NULL,\n        breaks = 20,\n        col = 'green',\n        probability = T)\n    \n    #generating density curve overlay\n    lines(density(x_omit), lwd = 2, col = \"dark blue\")\n    \n    #getting summary stats\n    five_num <- summary(x_omit)\n    #displaying five number summary (not showing the mean) \n    five_num <- as.matrix(five_num[-4])\n    t(five_num)\n}\n\n#creating four histograms \nhist_function(df$transactions, 'Transactions')\nhist_function(df$sales, 'Sales')\nhist_function(df$dcoilwtico, 'Oil Price')\nhist_function(df$onpromotion, 'On Promotion')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:21:18.500425Z","iopub.execute_input":"2022-07-21T03:21:18.502760Z","iopub.status.idle":"2022-07-21T03:21:21.924393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bar graph showing value counts of sales by department (family)\noptions(repr.plot.width = 14, repr.plot.height = 8)\ndf %>%\n  group_by(family) %>%\n  summarize(sales = sum(sales)) %>%\n  ggplot(aes(reorder(family, -sales), sales, fill = -sales)) +\n  geom_bar(stat = \"identity\") +\n  scale_y_continuous(labels = label_number(suffix = \"M\", scale  = 1e-06)) +\n  geom_text(aes(label = paste(round(sales/ 1e6, 1), \"M\")), cex = 4, vjust = .5, hjust = -.2, angle = 90)+\n  theme_bw() +\n  theme(axis.text.x = element_text(angle = 45, hjust = 1),\n        legend.position = 'none',\n        plot.title = element_text(hjust = 0.5, size = 24, face = \"bold\"),\n        plot.subtitle = element_text(hjust = 0.5, size = 14, face = 'italic'),\n        axis.title.x = element_text(size = 14),\n        axis.title.y = element_text(size = 14)) +\n  labs(title = 'Total Sales by Department (family)', subtitle = 'All departments', x = 'Department', y  = 'Sales')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:29:43.704656Z","iopub.execute_input":"2022-07-21T03:29:43.707924Z","iopub.status.idle":"2022-07-21T03:29:44.494769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bar graph showing departments with high sales\ndf %>%\n  group_by(family) %>%\n  summarize(sales = sum(sales)) %>%\n  filter(sales >= 1000000) %>%\n  ggplot(aes(reorder(family, -sales), sales, fill = -sales)) +\n  geom_bar(stat = \"identity\") +\n  scale_y_continuous(labels = label_number(suffix = \"M\", scale  = 1e-06)) +\n  geom_text(aes(label = paste(round(sales/ 1e6, 1), \"M\")), cex = 4, vjust = -.2)+\n  theme_bw() +\n  theme(axis.text.x = element_text(angle = 45, hjust = 1),\n        legend.position = 'none',\n        plot.title = element_text(hjust = 0.5, size = 24, face = \"bold\"),\n        plot.subtitle = element_text(hjust = 0.5, size = 14, face = 'italic'),\n        axis.title.x = element_text(size = 14),\n        axis.title.y = element_text(size = 14)) +\n  labs(title = 'Total Sales by Department (family)', subtitle = 'Showing departments with over $1mil sales', x = 'Department', y  = 'Sales')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:30:14.333741Z","iopub.execute_input":"2022-07-21T03:30:14.338345Z","iopub.status.idle":"2022-07-21T03:30:15.003273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bar graph showing departments with low sales\ndf %>%\n  group_by(family) %>%\n  summarize(sales = sum(sales)) %>%\n  filter(sales < 1000000) %>%\n  ggplot(aes(reorder(family, -sales), sales, fill = -sales)) +\n  geom_bar(stat = \"identity\") +\n  scale_y_continuous(labels = label_number(suffix = \"M\", scale  = 1e-06)) +\n  geom_text(aes(label = paste(round(sales/ 1e6, 1), \"M\")), cex = 4, vjust = -.2) +\n  theme_bw() +\n  theme(axis.text.x = element_text(angle = 45, hjust = 1),\n        legend.position = 'none',\n        plot.title = element_text(hjust = 0.5, size = 24, face = \"bold\"),\n        plot.subtitle = element_text(hjust = 0.5, size = 14, face = 'italic'),\n        axis.title.x = element_text(size = 14),\n        axis.title.y = element_text(size = 14)) +\n  labs(title = 'Total Sales by Department (family)', subtitle = 'Showing departments with under $1mil sales', x = 'Department', y  = 'Sales')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:30:54.615752Z","iopub.execute_input":"2022-07-21T03:30:54.617321Z","iopub.status.idle":"2022-07-21T03:30:55.079379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Value counts of sales by month\np1 <- df %>% \n  filter(Year != 2017) %>%\n  group_by(Monthofyear) %>%\n  summarize(sales = sum(sales), .groups = 'drop') %>%\n  ggplot(aes(x = Monthofyear, y = sales, fill = -sales)) +\n  geom_bar(stat = \"identity\") +\n  scale_y_continuous(labels = label_number(suffix = \"M\", scale  = 1e-06)) +\n  scale_fill_viridis_c() +\n  geom_text(aes(label = paste(round(sales/ 1e6, 1), \"M\")), cex = 4, vjust = -.2) +\n  theme_bw() +\n  theme(axis.text.x = element_text(angle = 45, hjust = 1),\n       plot.title = element_text(hjust = 0.5, size = 24, face = 'bold'),\n       plot.subtitle = element_text(hjust = 0.5, size = 14, face = 'italic'),\n       axis.title.x = element_text(size = 14),\n       axis.title.y = element_text(size = 14),\n       legend.position = 'none') +\n  labs(title = 'Total Sales by Month', subtitle = 'Excluding current year', x = 'Month', y = 'Sales') \n\n#Value counts of sales by day\np2 <- df %>% \n  group_by(Dayofweek) %>%\n  summarize(sales = sum(sales), .groups = 'drop') %>%\n  ggplot(aes(x = Dayofweek, y = sales, fill = -sales)) +\n  geom_bar(stat = \"identity\") +\n  scale_y_continuous(labels = label_number(suffix = \"M\", scale  = 1e-06)) +\n  geom_text(aes(label = paste(round(sales/ 1e6, 1), \"M\")), cex = 4, vjust = -.2) +\n  theme_bw() +\n  theme(axis.text.x = element_text(angle = 45, hjust = 1),\n       plot.title = element_text(hjust = 0.5, size = 24, face = 'bold'),\n       plot.subtitle = element_text(hjust = 0.5, size = 14, face = 'italic'),\n       axis.title.x = element_text(size = 14),\n       axis.title.y = element_text(size = 14),\n       legend.position = 'none') +\n  labs(title = 'Total Sales by Day', x = 'Day of week', y = 'Sales') \n\nplot_grid(plotlist = list(p1,p2), ncol = 2)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:32:46.555877Z","iopub.execute_input":"2022-07-21T03:32:46.557470Z","iopub.status.idle":"2022-07-21T03:32:48.130394Z"},"trusted":true},"execution_count":null,"outputs":[]}]}