# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(ggplot2) # Data visualization
library(readr) # CSV file I/O, e.g. the read_csv function

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

system("ls ../input")

# Any results you write to the current directory are saved as output.
library(data.table)
get_probs <- function (dt,var,target,w){
  p=dt[,sum(get(target))/.N]
  dt[ ,.( prob=(sum(get(target))+w*p )/(.N+w) ),by=eval(var)]
}
DT_fill_NA <- function(DT,replacement=0) {
  for (j in seq_len(ncol(DT)))
    set(DT,which(is.na(DT[[j]])),j,replacement)
}
super_fread <- function( file , key_var=NULL){
  dt <- fread(file)
  if(!is.null(key_var)) setkeyv(dt,c(key_var))
  return(dt)
}

clicks_train  <- super_fread( "../input/clicks_train.csv", key_var = "ad_id" )

click_prob = clicks_train[,.(sum(clicked)/.N)]
ad_id_probs   <- get_probs(clicks_train,"ad_id","clicked",8)
rm(clicks_train)
gc()

clicks_test   <- super_fread( "../input/clicks_test.csv" , key_var = "ad_id" )
clicks_test <- merge( clicks_test, ad_id_probs, all.x = T )

DT_fill_NA( clicks_test, click_prob )

setkey(clicks_test,"prob")
submission <- clicks_test[,.(ad_id=paste(rev(ad_id),collapse=" ")),by=display_id]
setkey(submission,"display_id")