library(ggplot2)
library(readr)
library(data.table)
library(Matrix)
library(xgboost)
library(caret)
library(dplyr)

set.seed(12345)
PATH <- "../input/"
system("ls ../input")
cat("Load data")
train <- fread(paste0(PATH,"train.csv"), sep=",", na.strings = "", stringsAsFactors=T)
transactions <- fread(paste0(PATH,"transactions.csv"), sep=",", na.strings = "", stringsAsFactors=T)
members <- fread(paste0(PATH,"members.csv"), sep=",", na.strings = "", stringsAsFactors=T)
sample_submission_zero <- fread(paste0(PATH,"sample_submission_zero.csv"), sep=",", na.strings = "", stringsAsFactors=T)

sample_submission_zero$is_churn <- NA
data <- rbind(train, sample_submission_zero)
data[,is_duplicate := as.numeric(duplicated(as.character(data$msno)) | duplicated(as.character(data$msno),fromLast=T))]
rm(train);gc()


members[,gender := as.numeric(gender)]
members$gender[is.na(members$gender)] <- 0

members[,":="(reg_fulldate = members$registration_init_time
             ,registration_init_time = as.Date(as.character(registration_init_time), '%Y%m%d')
             ,exp_fulldate = expiration_date
             ,expiration_date = as.Date(as.character(expiration_date), '%Y%m%d'))]
members[,":="(reg_year = year(registration_init_time)
             ,reg_month = month(registration_init_time)
             ,reg_mday = mday(registration_init_time)
             ,reg_wday = wday(registration_init_time)
             ,exp_year = year(expiration_date)
             ,exp_month = month(expiration_date)
             ,exp_mday = mday(expiration_date)
             ,exp_wday = wday(expiration_date)
             ,date_diff = as.numeric(expiration_date - registration_init_time))]
members <- subset(members, select = -c(registration_init_time, expiration_date))


data <- merge(data, members, by = "msno", all.x = TRUE)
rm(members);gc()


transactions <- transactions[transactions$msno %in% levels(data$msno),]


transactions[,n_transactions := .N, by = msno]


transactions[,payment_price_diff := plan_list_price - actual_amount_paid]

transactions <- transactions[,lapply(.SD,mean,na.rm=T), by = msno, .SDcols = names(transactions)[c(2:6,9:11)]]


data <- merge(data, transactions, by = "msno", all.x = TRUE)
rm(transactions);gc()


cvFolds <- createFolds(data$is_churn[!is.na(data$is_churn)], k=5, list=TRUE, returnTrain=FALSE)
varnames <- setdiff(colnames(data), c("msno", "is_churn"))
train_sparse <- Matrix(as.matrix(data[!is.na(is_churn), varnames, with=F]), sparse=TRUE)
test_sparse <- Matrix(as.matrix(data[is.na(is_churn), varnames, with=F]), sparse=TRUE)
y_train <- data[!is.na(is_churn),is_churn]
test_ids <- data[is.na(is_churn),msno]
dtrain <- xgb.DMatrix(data=train_sparse, label=y_train)
dtest <- xgb.DMatrix(data=test_sparse)


param <- list(booster="gbtree",
              objective="binary:logistic",
              eval_metric="logloss",
              eta = .02,
              gamma = 1,
              max_depth = 6,
              min_child_weight = 1,
              subsample = .8,
              colsample_bytree = .8
)

best_iter <- 1512

cat("xgb model")
xgb_model <- xgb.train(data = dtrain,
                       params = param,
                       watchlist = list(train = dtrain),
                       nrounds = best_iter,
                       verbose = 1,
                       print_every_n = 100
)

cat("Feature importance")
names <- dimnames(train_sparse)[[2]]
importance_matrix <- xgb.importance(names, model=xgb_model)
xgb.plot.importance(importance_matrix)

cat("Predict and output csv")
preds <- data.table(msno=test_ids, is_churn=predict(xgb_model,dtest))
write.table(preds, "submission1.csv", sep=",", dec=".", quote=FALSE, row.names=FALSE)
preds <- merge(sample_submission_zero[,1], preds, by="msno", all.x=T, sort=F)
write.table(preds, "submission.csv", sep=",", dec=".", quote=FALSE, row.names=FALSE)