## How do you pre-process?
## Work in progress; will add scaling,
## imputation, up/down sampling, etc.
## as applicable

## Note that you can add user logs by
## jagangupta, link below. I'm not sure
## how to upload the CSV file here. Tried
## adding data source, got errors.

##Load Libs


library(data.table)
library(caret)


### Load raw data


train.raw <- fread('../input/train.csv')

# Get train IDs and drop is_churn
train_ids <- as.data.frame(train.raw$msno)
train_is_churn <- as.data.frame(train.raw$is_churn)
train.raw[, is_churn := NULL]

members.raw <- fread('../input/members.csv')
transactions.raw <- fread('../input/transactions.csv')
submission.raw <- fread('../input/sample_submission_zero.csv')

# We'll get test IDs from submission file
test_ids <- as.data.frame(submission.raw$msno)
submission.raw[, is_churn := NULL]

# Thanks jagangupta!
# https://www.kaggle.com/jagangupta/processing-huge-datasets-user-log?scriptVersionId=1533548/output
# ul.raw <- fread('../input/user_logs_output.csv')


### Put Data Together


# Get most recent transaction
unique.trans <- transactions.raw[order(rank(transaction_date))]
unique.trans <- unique(transactions.raw, by=c("msno"))

setkey(train.raw, msno)
setkey(members.raw, msno)
setkey(transactions.raw, msno)
setkey(unique.trans, msno)
#setkey(ul.raw, msno)
setkey(submission.raw, msno)

# Combine into one big dataset
train1 <- members.raw[train.raw]
train <- unique.trans[train1]
#train <- ul.raw[train1]

train.count <- nrow(train)

test1 <- members.raw[submission.raw]
test <- unique.trans[test1]
#test <- ul.raw[test1]

combi <- rbind(train, test)

# Drop IDs, we'll add them back later
combi[,msno := NULL]

# Save some RAM

#rm("train.raw")
rm("members.raw")
rm("unique.trans")
#rm("ul.raw")
rm("submission.raw")
rm("train")
rm("test")
#rm("transactions.raw") # save for now, may make features with it


### First pass at reducing columns


# Remove vars with near zero variance
nzv_cols <- nearZeroVar(combi, names = TRUE)

for (z in nzv_cols){
                    combi[, eval(z) := NULL]
}


### Set data types, one hot encode, etc...


# Create factors where appropriate
factorlist <- c("city",
                "gender",
                "registered_via",
                "payment_method_id",
                #"payment_plan_days", # low variance
                "plan_list_price",
                "actual_amount_paid",
                "is_auto_renew"#,
                #"is_cancel" # low variance
                )

combi[, factorlist] <- lapply(combi[, factorlist, with=FALSE], factor)

# Encode factors
dmy <- dummyVars(" ~  
                city + 
                gender +
                registered_via + 
                payment_method_id + 
                plan_list_price + 
                actual_amount_paid + 
                is_auto_renew",
                data = combi, fullRank = T
)

e <- data.frame(predict(dmy, newdata = combi))

combi <- cbind(combi,e)

rm("e")

# Drop original factors

for (f in factorlist){
                combi[, eval(f) := NULL]
}


# Multicollinearity?

corrmatrix <- cor(combi[,1:ncol(combi)])
corrlist <- findCorrelation(corrmatrix, cutoff = .95, verbose = TRUE, names = TRUE, exact = TRUE)

for (c in corrlist){
                    combi[, eval(c) := NULL]
}

# Consider... normalize continuous data now or at training time?

# Split data into train and test
new_train = combi[1:train.count,]
new_test = combi[(train.count+1):nrow(combi),]

# Add IDs back

new_train <- cbind(train_ids,new_train)
new_train <- cbind(train_is_churn,new_train)

new_test <- cbind(test_ids,new_test)

# Here's a glance at our data...

head(combi)

str(combi)

summary(combi)

# Write new train and test files

write.csv(new_train, file = "new_train.csv")
write.csv(new_test, file = "new_test.csv")

# Performance tuning...
# Check for any objects that weren't removed but should have been
#ls()