# Edited...
# This kernel have improvement from Pranav Pandya and Andy Harless
# Pranav Kernel: https://www.kaggle.com/pranav84/xgboost-on-hist-mode-ip-addresses-dropped
# Andy Kernel: https://www.kaggle.com/aharless/jo-o-s-xgboost-with-memory-usage-enhancements

import gc
import time
import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split
import xgboost as xgb
from xgboost import plot_importance
import matplotlib.pyplot as plt

# Change this for validation with 10% from train
is_valid = False

path = '../input/'

def timeFeatures(df):
    # Make some new features with click_time column
    df['datetime'] = pd.to_datetime(df['click_time'])
    df['dow']      = df['datetime'].dt.dayofweek
    df["doy"]      = df["datetime"].dt.dayofyear
    #df["dteom"]    = df["datetime"].dt.daysinmonth - df["datetime"].dt.day
    df.drop(['click_time', 'datetime'], axis=1, inplace=True)
    return df

def before_and_after_clicks(x):
    HISTORY_CLICKS = {
        'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],
        'app_clicks': ['ip', 'app']}
    for name, set_ in HISTORY_CLICKS.items():
        x['prev_'+name] = x.groupby(set_).cumcount().rename('prev'+name)
        x['future_'+name] = x.iloc[::-1].groupby(set_).cumcount().rename('future'+name).iloc[::-1]
    return x
      
train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_columns  = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id']  

dtypes = {
        'ip'            : 'uint32',
        'app'           : 'uint16',
        'device'        : 'uint16',
        'os'            : 'uint16',
        'channel'       : 'uint16',
        'is_attributed' : 'uint8',
        'click_id'      : 'uint32'
        }

def procedure(x):
    # Read the last lines because they are more impacting in training than the starting lines
    if (x!=0):
        train = pd.read_csv(path+"train.csv", skiprows=range(1,x*20000000), nrows=2000000, usecols=train_columns, dtype=dtypes)
    else:
        train = pd.read_csv(path+"train.csv", nrows=20000000, usecols=train_columns, dtype=dtypes)
    test = pd.read_csv(path+"test_supplement.csv", usecols=test_columns, dtype=dtypes)

    print('Finished to load part data')
    train = before_and_after_clicks(train)
    gc.collect()
    # Drop the IP and the columns from target
    y = train['is_attributed']
    train.drop(['is_attributed'], axis=1, inplace=True)

    # Drop IP and ID from test rows
    
    #sub['click_id'] = test['click_id'].astype('int')
    test.drop(['click_id'], axis=1, inplace=True)
    gc.collect()

    nrow_train = train.shape[0]
    merge = pd.concat([train, test])

    del train, test
    gc.collect()

    # Count the number of clicks by ip
    ip_count = merge.groupby(['ip'])['channel'].count().reset_index()
    ip_count.columns = ['ip', 'clicks_by_ip']

    merge = pd.merge(merge, ip_count, on='ip', how='left', sort=False)
    merge['clicks_by_ip'] = merge['clicks_by_ip'].astype('uint16')
    merge.drop('ip', axis=1, inplace=True)

    train = merge[:nrow_train]
    test = merge[nrow_train:]

    del test, merge
    gc.collect()
    
    print('Start to generate time features')

    train = timeFeatures(train)
    gc.collect()
    
    return train, y, ip_count

print('Start XGBoost Training')
# Set the params(this params from Pranav kernel) for xgboost model
params = {'eta': 0.3,
        'tree_method': "hist",
        'grow_policy': "lossguide",
        'max_leaves': 1400,  
        'max_depth': 0, 
        'subsample': 0.9, 
        'colsample_bytree': 0.7, 
        'colsample_bylevel':0.7,
        'min_child_weight':0,
        'alpha':4,
        'objective': 'binary:logistic', 
        'scale_pos_weight':9,
        'eval_metric': 'auc', 
        'nthread':8,
        'random_state': 99, 
        'silent': True}

if (is_valid == True):
    # Get 10% of train dataset to use as validation
    train, y = procedure(123903891)
    x1, x2, y1, y2 = train_test_split(train, y, test_size=0.1, random_state=99)
    dtrain = xgb.DMatrix(x1, y1)
    dvalid = xgb.DMatrix(x2, y2)
    del x1, y1, x2, y2 
    gc.collect()
    watchlist = [(dtrain, 'train'), (dvalid, 'valid')]
    model = xgb.train(params, dtrain, 200, watchlist, maximize=True, early_stopping_rounds = 25, verbose_eval=5)
    del dvalid
else:
    for i in range(0, 4):
        train, y, ip_count = procedure(4)
        cols = list(train.columns)
        gc.collect()
        dtrain = xgb.DMatrix(train, y)
        del train, y
        gc.collect()
        watchlist = [(dtrain, 'train')]
        model = xgb.train(params, dtrain, 30, watchlist, maximize=True, verbose_eval=1)
        gc.collect()
        
del dtrain
gc.collect()

print('Finish XGBoost Training')

# Plot the feature importance from xgboost
plot_importance(model)
plt.gcf().savefig('feature_importance_xgb.png')

# Load the test for predict 
test = pd.read_csv(path+"test.csv", usecols=test_columns, dtype=dtypes)
test = before_and_after_clicks(test)
test = pd.merge(test, ip_count, on='ip', how='left', sort=False)
del ip_count
gc.collect()

sub = pd.DataFrame()
sub['click_id'] = test['click_id'].astype('int')

test['clicks_by_ip'] = test['clicks_by_ip'].astype('uint16')
test = timeFeatures(test)
gc.collect()

test.drop(['click_id', 'ip'], axis=1, inplace=True)
test = test[cols]
dtest = xgb.DMatrix(test)
del test
gc.collect()

# Save the predictions
sub['is_attributed'] = model.predict(dtest, ntree_limit=model.best_ntree_limit)
sub.to_csv('xgb_sub.csv', float_format='%.8f', index=False)