# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats

library(data.table)
library(readr)
library(caret)
library(stringdist)

# Read in data
location <- fread("../input/Location.csv")
itemPairsTest <- fread("../input/ItemPairs_test.csv")
itemPairsTrain <- fread("../input/ItemPairs_train.csv")
itemInfoTest <- read_csv("../input/ItemInfo_test.csv")
itemInfoTrain <- read_csv("../input/ItemInfo_train.csv")
itemInfoTest <- data.table(itemInfoTest)
itemInfoTrain <- data.table(itemInfoTrain)

setkey(location, locationID)
setkey(itemInfoTrain, itemID)
setkey(itemInfoTest, itemID)

# Drop unused factors
dropAndNumChar <- function(itemInfo){
  itemInfo[, ':=' (ncharTitle = nchar(title),
                   ncharDescription = nchar(description),
                   description = NULL,
                   images_array = NULL,
                   attrsJSON = NULL)]
}

dropAndNumChar(itemInfoTest)
dropAndNumChar(itemInfoTrain)

# Merge
mergeInfo <- function(itemPairs, itemInfo){
  # merge on itemID_1
  setkey(itemPairs, itemID_1)
  itemPairs <- itemInfo[itemPairs]
  setnames(itemPairs, names(itemInfo), paste0(names(itemInfo), "_1"))
  # merge on itemID_2
  setkey(itemPairs, itemID_2)
  itemPairs <- itemInfo[itemPairs]
  setnames(itemPairs, names(itemInfo), paste0(names(itemInfo), "_2"))
  # merge on locationID_1
  setkey(itemPairs, locationID_1)
  itemPairs <- location[itemPairs]
  setnames(itemPairs, names(location), paste0(names(location), "_1"))
  # merge on locationID_2
  setkey(itemPairs, locationID_2)
  itemPairs <- location[itemPairs]
  setnames(itemPairs, names(location), paste0(names(location), "_2"))
  return(itemPairs)
}

itemPairsTrain <- mergeInfo(itemPairsTrain, itemInfoTrain)
itemPairsTest <- mergeInfo(itemPairsTest, itemInfoTest)

rm(list=c("itemInfoTest", "itemInfoTrain", "location")); gc()

# Create features
matchPair <- function(x, y){
  ifelse(is.na(x), ifelse(is.na(y), 3, 2), ifelse(is.na(y), 2, ifelse(x==y, 1, 4)))
}

createFeatures <- function(itemPairs){
  itemPairs[, ':=' (priceAvg = (ifelse(is.na(price_1), price_2, price_1) + price_2)/2,
                    price_1 = NULL,
                    price_2 = NULL,
                    titleStringDist = stringdist(title_1, title_2, method = "jw"),
                    titleStringDist2 = (stringdist(title_1, title_2, method = "lcs") / 
                        pmax(ncharTitle_1, ncharTitle_2, na.rm=TRUE)),
                    title_1 = NULL,
                    title_2 = NULL,
                    ncharTitle_1 = NULL,
                    ncharTitle_2 = NULL,
                    descriptionAvg = (ifelse(is.na(ncharDescription_1), ncharDescription_2, ncharDescription_1) + ncharDescription_2)/2,
                    ncharDescription_1 = NULL,
                    ncharDescription_2 = NULL,
                    lat_1 = NULL,
                    lat_2 = NULL,
                    lon_1 = NULL,
                    lon_2 = NULL
                    # ,
                    # itemID_1 = NULL,
                    # itemID_2 = NULL
                    )]
  
#   itemPairs[, ':=' (priceAbs = ifelse(is.na(priceAbs), 0, priceAbs),
#                     priceDiff = ifelse(is.na(priceDiff), 0, priceDiff),
#                     priceMin = ifelse(is.na(priceMin), 0, priceMin),
#                     priceMax = ifelse(is.na(priceMax), 0, priceMax),
#                     titleStringDist = ifelse(is.na(titleStringDist), 0, titleStringDist),
#                     titleStringDist2 = ifelse(is.na(titleStringDist2) | titleStringDist2 == Inf, 0, titleStringDist2))]
}

createFeatures(itemPairsTrain); gc()
f <- gzfile("train_avg.csv.gz")
write.csv(itemPairsTrain, f, row.names=F)
rm(itemPairsTrain); gc()
createFeatures(itemPairsTest); gc()
f <- gzfile("test_avg.csv.gz")
write.csv(itemPairsTest, f, row.names=F)
rm(itemPairsTest); gc()
# write.csv(itemPairsTest, file='test.csv', row.names=F)

# library(xgboost)

# maxTrees <- 87
# shrinkage <- 0.09
# gamma <- 1
# depth <- 13
# minChildWeight <- 38
# colSample <- 0.4
# subSample <- 0.37
# earlyStopRound <- 4

# modelVars <- names(itemPairsTrain)[which(!(names(itemPairsTrain) %in% c("isDuplicate", "generationMethod", "foldId")))]

# itemPairsTest <- data.frame(itemPairsTest)
# itemPairsTrain <- data.frame(itemPairsTrain)


# set.seed(0)

# # Matrix
# dtrain <- xgb.DMatrix(as.matrix(itemPairsTrain[, modelVars]), label=itemPairsTrain$isDuplicate)
# dtest <- xgb.DMatrix(as.matrix(itemPairsTest[, modelVars]))

# xgbResult <- xgboost(params=list(max_depth=depth,
#                                  eta=shrinkage,
#                                  gamma=gamma,
#                                  colsample_bytree=colSample,
#                                  min_child_weight=minChildWeight),
#                      data=dtrain,
#                      nrounds=90,
#                      objective="binary:logistic",
#                      eval_metric="auc")

# testPreds <- predict(xgbResult, dtest)

# submission <- data.frame(id=itemPairsTest$id, probability=testPreds)
# write.csv(submission, file="submission.csv",row.names=FALSE)