if (!require("pacman")) install.packages("pacman")
p_load(knitr, pryr, tidyverse, data.table, tictoc, fasttime, lubridate, caret, DescTools, lightgbm)
set.seed(84)
options(scipen = 9999, warn = -1, digits= 4)

tic("total processing time for 40 mln observations: ")
train_path <- "../input/train.csv"

vars <- c("ip", "app", "device", "os", "channel", "click_time", "attributed_time", "is_attributed")
most_freq_hours_in_test_data <- c("4","5","9","10","13","14")
least_freq_hours_in_test_data <- c("6","11","15")

# features builder 
add_features <- function(df) {
  cat("Addidng features..", "\n")
      df <- as.data.table(df)
      df[, day:=              Weekday(click_time)]
      df[, hour:=             hour(click_time)]
      df[, numeric_ct:=       as.numeric(fastPOSIXct(click_time))]
      df[, UsrappCount:=.N,   by=list(ip, app, device, os)]
      df[, tlag:=             as.numeric(numeric_ct-lag(numeric_ct), units = 'secs'),by=UsrappCount]
      df$click_time <- NULL   # Litlle adjustments to stay within memory limits
      df$numeric_ct <- NULL
      df[, in_test_hh:=       (ifelse(hour %in% most_freq_hours_in_test_data, 1, 
                               ifelse(hour %in% least_freq_hours_in_test_data, 3, 2)))]
      df$in_test_hh <- NULL
      df[, nip_dth:=.N,       by=list(ip, day, in_test_hh)]
      df[, nos_dth:=.N,       by=list(os, day, in_test_hh)]
      df[, napp_dth:=.N,      by=list(app, day, in_test_hh)]
      df$in_test_hh <- NULL
      df[, nip:=.N,           by=list(ip, day, hour)]
      df[, nipApp:=.N,        by=list(ip, day, hour, app)]
      df[, nipOs:=.N,         by=list(ip, day, hour, os)]
      df[, nipAppOs:=.N,      by=list(ip, day, hour, app, os)]
      df[, napp:=.N,          by=list(app, day, hour)]
      df[, nappDevOs:=.N,     by=list(app, day, device, os)]
      df$day <- NULL
      df[, UsrappRank:=1:.N,  by=list(ip, app, device, os)]
      df[, UsrRank:=1:.N,     by=list(ip, os, device)]
      df <- as.data.frame(df) %>% select(-c(ip))
  return(df)
}

cat("
--------------------------------------------------------------
            Part 1: Reading training data 
--------------------------------------------------------------
")

total_rows <- 184903890
chunk_rows <- 40000000
skip_rows  <- total_rows - chunk_rows
train <- fread(train_path, skip=skip_rows, nrows= chunk_rows, col.names = vars,
               colClasses=list(numeric=1:5), showProgress = FALSE) %>% 
         select(-c(attributed_time))
head(train)
cat("
--------------------------------------------------------------
            Part 2: Validation partition
--------------------------------------------------------------
")

train.index <- createDataPartition(train$is_attributed, p = 0.9, list = FALSE)
dtrain <- train[ train.index,]
dvalid <- train[-train.index,]
rm(train, train.index, total_rows, skip_rows)
invisible(gc())
cat("train size : ", dim(dtrain), "\n")
cat("valid size : ", dim(dvalid), "\n")
    
cat("
--------------------------------------------------------------
            Part 3: Feature Engineering
--------------------------------------------------------------
")

dtrain <- add_features(dtrain)
print("features: ")
names(dtrain)
print("Table of class unbalance in train")
table(dtrain$is_attributed)
cat("train size : ", dim(dtrain), "\n")

dtrain <- lgb.Dataset(data  = as.matrix(dtrain[, colnames(dtrain) != "is_attributed"]),
                      label = dtrain$is_attributed, 
                      categorical_feature = c("app", "device", "os", "channel", "hour"))
invisible(gc())

dvalid <- add_features(dvalid)
cat("valid size : ", dim(dvalid), "\n")
print("Table of class unbalance in validation split")
table(dvalid$is_attributed)
dvalid <- lgb.Dataset(data  = as.matrix(dvalid[, colnames(dvalid) != "is_attributed"]),
                      label = dvalid$is_attributed, 
                      categorical_feature = c("app", "device", "os", "channel", "hour"))
invisible(gc())
toc()

cat("
--------------------------------------------------------------
            Part 4: Modelling
--------------------------------------------------------------
")

params = list(objective = "binary", metric = "auc", learning_rate= 0.1, num_leaves= 7, max_depth= 4,
              min_child_samples= 100, max_bin= 100, subsample= 0.7, subsample_freq= 2, colsample_bytree= 0.7,
              min_child_weight= 0, min_split_gain= 0, scale_pos_weight= 99.7)
              
model <- lgb.train(params, dtrain, valids = list(validation = dvalid), nthread = 4,
                   nrounds = 2000,verbose= 1, early_stopping_rounds = 50, eval_freq = 100)

cat("--------------------------------", "\n")
cat("Validation AUC: ", max(unlist(model$record_evals[["validation"]][["auc"]][["eval"]])), "\n")
cat("--------------------------------", "\n")
lgb.save(model, "model")

rm(list=ls()) #clear EVERYTHING from workspace 


cat("
--------------------------------------------------------------
            Part 5: Processing test data
--------------------------------------------------------------
")

if (!require("pacman")) install.packages("pacman")
p_load(knitr, pryr, tidyverse, data.table, tictoc, fasttime, lubridate, caret, DescTools, lightgbm)
set.seed(84)
options(scipen = 9999, warn = -1, digits= 4)

tic("total processing time for test data: ")
vars <- c("ip", "app", "device", "os", "channel", "click_time", "attributed_time", "is_attributed")
most_freq_hours_in_test_data <- c("4","5","9","10","13","14")
least_freq_hours_in_test_data <- c("6","11","15")

# features builder 
add_features <- function(df) {
  cat("Addidng features..", "\n")
      df <- as.data.table(df)
      df[, day:=              Weekday(click_time)]
      df[, hour:=             hour(click_time)]
      df[, numeric_ct:=       as.numeric(fastPOSIXct(click_time))]
      df[, UsrappCount:=.N,   by=list(ip, app, device, os)]
      df[, tlag:=             as.numeric(numeric_ct-lag(numeric_ct), units = 'secs'),by=UsrappCount]
      df$click_time <- NULL   # Litlle adjustments to stay within memory limits
      df$numeric_ct <- NULL
      df[, in_test_hh:=       (ifelse(hour %in% most_freq_hours_in_test_data, 1, 
                               ifelse(hour %in% least_freq_hours_in_test_data, 3, 2)))]
      df[, nip_dth:=.N,       by=list(ip, day, in_test_hh)]
      df[, nos_dth:=.N,       by=list(os, day, in_test_hh)]
      df[, napp_dth:=.N,      by=list(app, day, in_test_hh)]
      df$in_test_hh <- NULL
      df[, nip:=.N,           by=list(ip, day, hour)]
      df[, nipApp:=.N,        by=list(ip, day, hour, app)]
      df[, nipOs:=.N,         by=list(ip, day, hour, os)]
      df[, nipAppOs:=.N,      by=list(ip, day, hour, app, os)]
      df[, napp:=.N,          by=list(app, day, hour)]
      df[, nappDevOs:=.N,     by=list(app, day, device, os)]
      df$day <- NULL
      df[, UsrappRank:=1:.N,  by=list(ip, app, device, os)]
      df[, UsrRank:=1:.N,     by=list(ip, os, device)]
      df <- as.data.frame(df) %>% select(-c(ip))
  return(df)
}

test <- fread("../input/test.csv", colClasses=list(numeric=2:6), showProgress = FALSE)
sub  <- data.table(click_id = test$click_id, is_attributed = NA)
test$click_id <- NULL
test <- add_features(test)
test <- as.matrix(test[, colnames(test)])
invisible(gc())

cat("
--------------------------------------------------------------
            Part 7: Predictions and submission 
--------------------------------------------------------------
")

model <- lgb.load("model")
print("Preparing submission file...")
sub$is_attributed <- round(as.data.frame(predict(model, data = test, n = model$best_iter)),4)

head(sub)
fwrite(sub, "sub_LGB_40mln_R.csv")

print("Feature importance")
kable(lgb.importance(model, percentage = TRUE))
lgb.plot.importance(lgb.importance(model, percentage = TRUE), measure = "Gain")