library(data.table)

events <- fread('../input/events.csv')

views  <- fread('../input/page_views_sample.csv')
views  <- views[ , .N , keyby=c("document_id","uuid") ] 

events[, leak := views[J(events$document_id,events$uuid)]$N  ]
setkeyv( events, c("display_id","document_id") )
rm(views);gc() 

####Load Train and Test
train<- fread( "../input/clicks_train.csv" )
test <- fread( "../input/clicks_test.csv" )

####Map document_id to train and test
promoted <- fread( "../input/promoted_content.csv" )
setkeyv( promoted , "ad_id" )
#train[, document_id := promoted[J(train$ad_id)]$document_id ] 
test[ , document_id := promoted[J(test$ad_id )]$document_id ] 
rm(promoted); gc()


####Calculate ad_id probabilities
prob_by_adid <- train[,list( mean(clicked), length(clicked) ),keyby="ad_id"]; gc()
M <- mean( train$clicked)
prob_by_adid[, likelihood := ((V1*V2)+(12*M))/(12+V2)  ]
print( head(prob_by_adid) )


####In-fold validation
#train[             , prob := prob_by_adid[J(train$ad_id)]$likelihood ]; gc()#Map probabilities
#head(train,10)
#train[             , leak := events[J(train$display_id,train$document_id)]$leak ]; gc()
#head(train,10)
#print( mean( !is.na(train$leak), na.rm=T ) )
#train[ !is.na(leak), prob := 1 ]#Replace prob by leak
#setorderv( train, c("display_id","prob") , c(1,-1) )
#train[             , sort := 1:.N , by="display_id" ]; gc()
#print( mean( train[, sum(clicked/sort) , by="display_id" ]$V1 ) ); gc()
#head(train,20)
rm(train);gc()


####Build test predictions
print("Now Map testset probs")
test[             , prob := prob_by_adid[J(test$ad_id)]$likelihood ]#Map probabilities
test[ is.na(prob) , prob := M ]#Remove NAs
test[             , leak := events[J(test$display_id,test$document_id)]$leak ]#Map leak
test[ !is.na(leak), prob := 2 ]#Replace prob by leak
print( head(test,20) )
print( mean( !is.na(test$leak) ) )
rm(prob_by_adid, events);gc()


print("Now Build the Submission")
setorderv( test, c("display_id","prob"), c(1,-1)  );gc() #Sort by -prob
sub <- test[,.(ad_id=paste0(ad_id,collapse=" ")), keyby="display_id" ];gc()#Build submission

write.csv(sub,file = "submission_leak1.csv",row.names = F)