library(lightgbm)
library(data.table)

set.seed(3)

#---------------------------
cat("Loading data...\n")

dt <- fread("../input/train.csv", drop = "MachineIdentifier")
dt <- dt[sample(.N, .N %/% 2), ]

y <- dt[, HasDetections]
dt[, HasDetections := NULL]
N <- dt[, .N]

dt <- rbind(dt, fread("../input/test.csv", drop = "MachineIdentifier"))

#---------------------------
cat("Adding group features...\n")

dt[, f1 := .N, by = "AvSigVersion,Wdft_IsGamer,Census_OSInstallLanguageIdentifier"
  ][, f2 := .N, by = "Census_ProcessorCoreCount,Wdft_RegionIdentifier,Census_OSBuildNumber"
  ][, f3 := .N, by = "Census_ProcessorCoreCount,Census_OEMNameIdentifier,CityIdentifier"
  ][, f4 := .N, by = "GeoNameIdentifier,Census_OEMNameIdentifier,Census_OSBuildRevision"]

#---------------------------
cat("Converting columns...\n")

cats <- names(which(sapply(dt, is.character)))
dt[, (cats) := lapply(.SD, function(x) as.integer(as.factor(x))), .SDcols = cats]  

dt <- data.matrix(dt)

rm(cats); invisible(gc())

#---------------------------
cat("Preparing data for boosting...\n")

tr <- lgb.Dataset(data = dt[1:N, ], label = y)
te <- dt[-(1:N), ]

rm(dt, y, N); invisible(gc())

#---------------------------
cat("Training and predicting...\n")

subm <- fread("../input/sample_submission.csv")
subm[, HasDetections := 0]

n_bags <- 3
for (i in seq(0.7, 0.9, length.out = n_bags)) {
  p <- list(boosting_type = "gbdt", 
            objective = "binary",
            metric = "auc", 
            nthread = 4, 
            learning_rate = 0.05, 
            max_depth = 5,
            num_leaves = 20,
            sub_feature = i, 
            sub_row = i, 
            bagging_freq = 1,
            lambda_l1 = 0.1, 
            lambda_l2 = 0.1)
  
  m_gbm <- lgb.train(p, tr, 5500,  verbose = -1)
  subm[, HasDetections := HasDetections + predict(m_gbm, te) / n_bags]
  
  rm(m_gbm); invisible(gc())
}

#---------------------------
cat("Making submission file...\n")

fwrite(subm, "ms_malware.csv")