# Load libraries
cat("step 1. libraries loading")
pkgs <- c("data.table", 
          "tidyverse", 
          "forcats", 
          "lubridate", 
          "stringr", 
          "rpart", 
          "randomForest", 
          "gbm", 
          "gmodels",
          "broom",
          "caret")

sapply(pkgs, require, character.only = T)

cat("step 2. data import & preprocessing")
# Chapter 2. Data Import & Preprocessing
# test data
test <- fread("../input/sample_submission_v2.csv")

# train data
train <- fread("../input/train_v2.csv") # 970960

# members data
members <- fread("../input/members_v3.csv") # 6769473
members <- members %>% filter(bd > 0 & bd < 100)

# transactions data
transactions <- fread("../input/transactions_v2.csv") # 1431009

# user_logs data
user_logs <- fread("../input/user_logs_v2.csv") # 18396362

# Chapter 3.1. Data Table Joining / Train data
cat("data joining train & members")
tr_members <- train %>% left_join(members, by = "msno")

cat("data joining train & members & transactions")
trm_transactions <- tr_members %>% 
  left_join(transactions, by = "msno")

cat("data joining train & members & transactions & userlogs")
trmt_userlogs <- trm_transactions %>%  
  left_join(user_logs, by = "msno")

cat("extract recent training date only")
training <- trmt_userlogs %>% 
  arrange(desc(date)) %>% 
  group_by(msno, is_churn, city, gender, registered_via, payment_method_id, is_auto_renew, is_cancel) %>% 
  summarise(
    bd = max(bd), 
    registration_init_time = max(registration_init_time),
    payment_plan_days = max(payment_plan_days), 
    plan_list_price = max(plan_list_price), 
    actual_amount_paid = max(actual_amount_paid), 
    transaction_date = max(transaction_date),
    membership_expire_date = max(membership_expire_date), 
    date = max(date),
    num25_total = sum(num_25), 
    num50_total = sum(num_50), 
    num75_total = sum(num_75), 
    num985_total = sum(num_985), 
    num100_total = sum(num_100), 
    numunq_total = sum(num_unq), 
    total_secs = sum(total_secs)
  )
 cat("The END of Training Data Joining")

cat("# save RAM")
rm(list = "tr_members","trm_transactions", "trmt_userlogs")


# Chapter 3.2. Data Table Joining / Test data
cat("data joining test & members")
te_members <- test %>% 
  left_join(members, by = "msno")

cat("data joining test & members & transactions")
tem_transactions <- te_members %>% 
  left_join(transactions, by = "msno")

cat("data joining test & members & transactions & userlogs")
temt_userlogs <- tem_transactions %>%  
  left_join(user_logs, by = "msno")

cat("extract recent testing date only")
testing <- temt_userlogs %>% 
  arrange(desc(date)) %>% 
  group_by(msno, is_churn, city, gender, registered_via, payment_method_id, is_auto_renew, is_cancel) %>% 
  summarise(
    bd = max(bd), 
    registration_init_time = max(registration_init_time),
    payment_plan_days = max(payment_plan_days), 
    plan_list_price = max(plan_list_price), 
    actual_amount_paid = max(actual_amount_paid), 
    transaction_date = max(transaction_date),
    membership_expire_date = max(membership_expire_date), 
    date = max(date),
    num25_total = sum(num_25), 
    num50_total = sum(num_50), 
    num75_total = sum(num_75), 
    num985_total = sum(num_985), 
    num100_total = sum(num_100), 
    numunq_total = sum(num_unq), 
    total_secs = sum(total_secs)
  )

# save RAM
cat("# save RAM")
rm(list = "te_members", "tem_transactions", "temt_userlogs")

cat("glimpse Training / Testing")
glimpse(training)
glimpse(testing)

testing <- tbl_df(testing)
training <- tbl_df(training)
cat("data type conversion / testing data")

cat("answer msno")
answer <- testing$msno

glimpse(answer)

test <- testing %>% 
    select(-c(msno, is_churn)) %>% 
    mutate(
        city = as.factor(city), 
        gender = as.factor(gender), 
        registered_via = as.factor(registered_via), 
        payment_method_id = as.factor(registered_via), 
        is_auto_renew = as.factor(is_auto_renew), 
        is_cancel = as.factor(is_cancel), 
        registration_init_time = ymd(registration_init_time), 
        transaction_date = ymd(transaction_date), 
        membership_expire_date = ymd(membership_expire_date), 
        date = ymd(date)
  )

cat("data type conversion / training data")
train <- training %>% 
    select(-msno) %>% 
    mutate(
        TARGET = factor(is_churn),
        city = as.factor(city), 
        gender = as.factor(gender), 
        registered_via = as.factor(registered_via), 
        payment_method_id = as.factor(registered_via), 
        is_auto_renew = as.factor(is_auto_renew), 
        is_cancel = as.factor(is_cancel), 
        registration_init_time = ymd(registration_init_time), 
        transaction_date = ymd(transaction_date), 
        membership_expire_date = ymd(membership_expire_date), 
        date = ymd(date), 
  )

cat("glimpse train / test")
glimpse(train)
glimpse(test)

cat("# step 3. imbalanced data - upSample")
set.seed(1121)
cat("imbalanced data")
prop.table(table(train$is_churn))

cat("balanced data")
up_train <- upSample(x = train[ , -23], y = train$TARGET)
prop.table(table(up_train$Class))
train2 <- up_train %>% select(-is_churn)

cat("# step 4. Modeling")
model <- glm(Class ~. -payment_method_id, data = train2, family = "binomial")

# Call summary
summary(model)
# Call glance
perf <- glance(model)

# Calculate pseudo-R-squared
pseudoR2 <- 1-(perf$deviance / perf$null.deviance)
pseudoR2

model$xlevels[["payment_method_id"]] <- union(model$xlevels[["payment_method_id"]], levels(test$payment_method_id))

cat("# step 5. Prediction")
pred <- predict(model, newdata = test, type = "response")

cat("# step 6. Answersheet")
submission <- data.frame(msno = answer, is_churn = round(pred))
submission[is.na(submission)] <- 0
submission$msno <- str_replace(submission$msno, pattern = "//.", "")
write.csv(submission, "submission.csv", row.names=FALSE)