#!/usr/bin/env python3 import gc import sys # python2 compatibility if sys.version_info[0] == 3: unicode = str import catboost as cb import pandas as pd import numpy as np import sklearn.model_selection # configuration DEBUG = False # test on small sample for debugging PROCESSED_DATA_NAME = 'simple1' # name for saving preprocesed data OUTPUT_NAME = 'simple1_full' # name for final results - model and predictions on test set INPUT_PATH = '../input/' SAVE_PREPROCESSED_DATA = False # False because of Kaggle's limited disk space, prefer True when running locally USE_PREPROCESSED_DATA = False # use stored preprocessed data, don't rerun preprocessing SAVE_MODEL = False # False because of Kaggle's limited disk space, prefer True when running locally VALIDATE = False # run with validation instead of using all train set for training TRAIN_SHARE = 0.9 # share of validation set from all training set CB_TECH_PARAMS = { 'verbose' : True, 'random_seed' : 42, 'save_snapshot' : False, # False because of Kaggle's limited disk space, prefer True when running locally 'allow_writing_files' : False, # False because of Kaggle's limited disk space, prefer True when running locally 'used_ram_limit' : 5*(2 ** 30), } CB_TRAIN_PARAMS = { 'eval_metric' : 'RMSE' } CAT_FEATURE_NAMES = [ 'user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type', # computed 'has_image', 'activation_date_weekday', ] NON_CAT_FEATURE_NAMES = [ 'price', 'item_seq_number', # computed 'title_length', 'description_length' ] TARGET_NAME = 'deal_probability' DEBUG_SIZE = 50000 ################################################################ NAME_PREFIX = 'debug.' if DEBUG else '' PROCESSED_DATA_PREFIX = NAME_PREFIX + PROCESSED_DATA_NAME + '.' OUTPUT_PREFIX = NAME_PREFIX + OUTPUT_NAME + '.' CB_TECH_PARAMS['snapshot_file'] = OUTPUT_PREFIX + 'catboost_snapshot' CB_PARAMS = CB_TECH_PARAMS CB_PARAMS.update(CB_TRAIN_PARAMS) FEATURE_NAMES = CAT_FEATURE_NAMES + NON_CAT_FEATURE_NAMES ################################################################ # prefix is either 'train' or 'test' # returns X, Y, None for 'train' # returns X, None, item_ids for 'test' def preprocess_data(prefix): def log(msg): print ('preprocess_data. prefix "', prefix, '" ', msg) log('start') cols = { #'item_id' 'user_id': str, 'region': unicode, 'city': unicode, 'parent_category_name': unicode, 'category_name': unicode, 'param_1': unicode, 'param_2': unicode, 'param_3': unicode, 'title' : unicode, 'description' : unicode, 'price' : np.float32, 'item_seq_number': np.uint32, 'activation_date': object, # in fact yyyy-mm-dd date 'user_type': str, 'image': str, 'image_top_1': np.float32, } if prefix == 'train': cols['deal_probability'] = np.float32 elif prefix == 'test': cols['item_id'] = str else: raise Exception('prefix is neither train nor test') df = pd.read_csv( INPUT_PATH + prefix + '.csv', nrows=DEBUG_SIZE if DEBUG else None, usecols=cols.keys(), dtype=cols, encoding = 'utf8' ) log('data after loading') print (df.info()) print (df.head(5)) # comp features # nans in 'image' column are parsed as floats df['has_image'] = df.image.apply(lambda image: True if type(image) == unicode else False).astype('bool') df.drop(['image'], axis=1, inplace=True) gc.collect() df['activation_date_weekday'] = pd.to_datetime(df.activation_date).apply(lambda ad: ad.weekday()).astype('uint8') df.drop(['activation_date'], axis=1, inplace=True) gc.collect() for col in ['title', 'description']: df[col + '_length'] = df[col].apply(lambda txt: len(txt) if type(txt) == unicode else 0).astype('uint32') df.drop([col], axis=1, inplace=True) gc.collect() # fix for catboost's python2 version unicode incompatibility for col, coltype in cols.items(): if (coltype == unicode) and (col in df): df[col] = df[col].apply( lambda utxt: utxt.encode('ascii', 'xmlcharrefreplace') if type(utxt) == unicode else utxt ).astype('str') gc.collect() log('data after comp features') print (df.info()) print (df.head(5)) log('end') return (df[FEATURE_NAMES].values, (df[TARGET_NAME].values if prefix == 'train' else None), (df['item_id'].values if prefix == 'test' else None)) # prefix is either 'train' or 'test' # returns X, Y, None for 'train' # returns X, None, item_ids for 'test' def get_preprocessed_data(prefix): pp_prefix = PROCESSED_DATA_PREFIX + prefix + '.' if USE_PREPROCESSED_DATA: return (np.load(pp_prefix + 'X.npy'), (np.load(pp_prefix + 'Y.npy') if prefix == 'train' else None), (np.load(pp_prefix + 'item_ids.npy') if prefix == 'test' else None)) else: X, Y, item_ids = preprocess_data(prefix) if SAVE_PREPROCESSED_DATA: for name in ('X', 'Y') if prefix == 'train' else ('X', 'item_ids'): np.save(pp_prefix + name + '.npy', eval(name)) return X, Y, item_ids """ returns (cb_model, val_pool) val_pool can be useful later of additional metrics evaluation pass None as val_X and val_Y to train w/o eval_set """ def train_model(train_X, train_Y, val_X, val_Y, feature_names, cat_feature_names): print ('start training') print ('feature_names', feature_names) print ('cat_feature_names', cat_feature_names) cat_feature_indices = [feature_names.index(cat_name) for cat_name in cat_feature_names] train_pool = cb.Pool( train_X, label=train_Y, feature_names=feature_names, cat_features=cat_feature_indices ) if (val_X is not None) and (val_Y is not None): val_pool = cb.Pool( val_X, label=val_Y, feature_names=feature_names, cat_features=cat_feature_indices ) else: val_pool = None cb_model = cb.CatBoostRegressor(**CB_PARAMS) cb_model.fit(train_pool, eval_set=val_pool) if SAVE_MODEL: cb_model.save_model(OUTPUT_PREFIX + 'model.cbm') del train_pool gc.collect() print ('end training') return (cb_model, val_pool) def predict(item_ids, test_X, cb_model): print ('start predict') sub = pd.DataFrame() sub['item_id'] = item_ids sub['deal_probability'] = np.clip( cb_model.predict(test_X), 0.0, 1.0) sub.to_csv(OUTPUT_PREFIX + 'submission.csv', index=False) print ('end predict') ################################################################ alltrain_X, alltrain_Y, _ = get_preprocessed_data('train') if VALIDATE: train_X, val_X, train_Y, val_Y = sklearn.model_selection.train_test_split( alltrain_X, alltrain_Y, train_size=TRAIN_SHARE, random_state=99 ) else: train_X, val_X, train_Y, val_Y = alltrain_X, None, alltrain_Y, None cb_model, _ = train_model(train_X, train_Y, val_X, val_Y, FEATURE_NAMES, CAT_FEATURE_NAMES) test_X, _, item_ids = get_preprocessed_data('test') predict(item_ids, test_X, cb_model)