# Simple first attempt to predict the propability of demand
# Not using the image info so far and only taking simple 
# categorical features into account
#
import numpy as np
import pandas as pd
import math
import time
import os.path
import gc
import random

import xgboost as xgb

def print_duration (start_time, msg):
    print("[%d] %s" % (int(time.time() - start_time), msg))
    start_time = time.time()
    return start_time
    
# quick way of calculating a numeric has for a string
def n_hash(s):
    random.seed(hash(s))
    return random.random()

# hash a complete column of a pandas dataframe    
def hash_column (row, col):
    if col in row:
        return n_hash(row[col])
    return n_hash('none')

def main():
    start_time = time.time()
    # create a xgboost model
    model = xgb.XGBRegressor(n_estimators=400, learning_rate=0.05, gamma=0, subsample=0.75, colsample_bytree=1, max_depth=7)
    
    # load the training data
    train = pd.read_csv('../input/train.csv')
    # calculate consistent numeric hashes for any categorical features 
    train['user_hash'] = train.apply (lambda row: hash_column (row, 'user_id'),axis=1)
    train['region_hash'] = train.apply (lambda row: hash_column (row, 'region'),axis=1)
    train['city_hash'] = train.apply (lambda row: hash_column (row, 'city'),axis=1)
    train['parent_category_name_hash'] = train.apply (lambda row: hash_column (row, 'parent_category_name'),axis=1)
    train['category_name_hash'] = train.apply (lambda row: hash_column (row, 'category_name'),axis=1)
    train['user_type_hash'] = train.apply (lambda row: hash_column (row, 'user_type'),axis=1)
    # for the beginning I use only the information if there is an image or not 
    train['image_exists'] = train['image'].isnull().astype(int)
    # calc log for price to reduce effect of very large price differences
    train['price'] = np.log(train['price'])
    #print(train.groupby(['image_exists']).image_exists.count())
    #print(train['image_exists'])
    start_time = print_duration (start_time, "Finished reading")   

    # start training
    train_X = train.as_matrix(columns=['image_top_1', 'user_hash', 'price', 'region_hash', 'city_hash', 'parent_category_name_hash', 'category_name_hash', 'user_type_hash', 'image_exists'])
    model.fit(train_X, train['deal_probability'])
    
    # read test data set
    test = pd.read_csv('../input/test.csv')
    test['user_hash'] = test.apply (lambda row: hash_column (row, 'user_id'),axis=1)
    test['region_hash'] = test.apply (lambda row: hash_column (row, 'region'),axis=1)
    test['city_hash'] = test.apply (lambda row: hash_column (row, 'city'),axis=1)
    test['parent_category_name_hash'] = test.apply (lambda row: hash_column (row, 'parent_category_name'),axis=1)
    test['category_name_hash'] = test.apply (lambda row: hash_column (row, 'category_name'),axis=1)
    test['user_type_hash'] = test.apply (lambda row: hash_column (row, 'user_type'),axis=1)
    test['image_exists'] = test['image'].isnull().astype(int)
    test['price'] = np.log(test['price'])
    test_X = test.as_matrix(columns=['image_top_1', 'user_hash', 'price', 'region_hash', 'city_hash', 'parent_category_name_hash', 'category_name_hash', 'user_type_hash', 'image_exists'])
    start_time = print_duration (start_time, "Finished training, start prediction")   
    # predict the propabilities for binary classes    
    pred = model.predict(test_X)
    
    start_time = print_duration (start_time, "Finished prediction, start store results")    
    submission = pd.read_csv("../input/sample_submission.csv")
    submission['deal_probability'] = pred
    print(submission[submission['deal_probability'] > 0])
    submission.to_csv("submission.csv", index=False)
    start_time = print_duration(start_time, "Finished to store result")
    
if __name__ == '__main__':
    main()