import pandas as pd import time import numpy as np from sklearn.model_selection import train_test_split import lightgbm as lgb import gc import matplotlib.pyplot as plt # function to calculate next click def do_next_Click( df,agg_suffix='nextClick', agg_type='float32'): print(f">> \nExtracting {agg_suffix} time calculation features...\n") GROUP_BY_NEXT_CLICKS = [ # V1 # {'groupby': ['ip']}, # {'groupby': ['ip', 'app']}, # {'groupby': ['ip', 'channel']}, # {'groupby': ['ip', 'os']}, # V3 {'groupby': ['ip', 'app', 'device', 'os', 'channel']}, {'groupby': ['ip', 'os', 'device']}, {'groupby': ['ip', 'os', 'device', 'app']} ] # Calculate the time to next click for each group for spec in GROUP_BY_NEXT_CLICKS: # Name of new feature new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix) # Unique list of features to select all_features = spec['groupby'] + ['click_time'] # Run calculation print(f">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}") df[new_feature] = (df[all_features].groupby(spec[ 'groupby']).click_time.shift(-1) - df.click_time).dt.seconds.astype(agg_type) gc.collect() return (df) def do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'): print(f">> \nExtracting {agg_suffix} time calculation features...\n") GROUP_BY_NEXT_CLICKS = [ # V1 # {'groupby': ['ip']}, # {'groupby': ['ip', 'app']}, {'groupby': ['ip', 'channel']}, {'groupby': ['ip', 'os']}, # V3 #{'groupby': ['ip', 'app', 'device', 'os', 'channel']}, #{'groupby': ['ip', 'os', 'device']}, #{'groupby': ['ip', 'os', 'device', 'app']} ] # Calculate the time to next click for each group for spec in GROUP_BY_NEXT_CLICKS: # Name of new feature new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix) # Unique list of features to select all_features = spec['groupby'] + ['click_time'] # Run calculation print(f">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}") df[new_feature] = (df.click_time - df[all_features].groupby(spec[ 'groupby']).click_time.shift(+1) ).dt.seconds.astype(agg_type) gc.collect() return (df) # function to count by groups def do_count( df, group_cols, agg_type='uint32', show_max=False, show_agg=True ): agg_name='{}count'.format('_'.join(group_cols)) if show_agg: print( "\nAggregating by ", group_cols , '... and saved in', agg_name ) gp = df[group_cols][group_cols].groupby(group_cols).size().rename(agg_name).to_frame().reset_index() df = df.merge(gp, on=group_cols, how='left') del gp gc.collect() if show_max: print( agg_name + " max value = ", df[agg_name].max() ) df[agg_name] = df[agg_name].astype(agg_type) # print('predictors',predictors) gc.collect() return( df ) # function to count unique ips def do_countuniq( df, group_cols, counted, agg_type='uint32', show_max=False, show_agg=True ): agg_name= '{}_by_{}_countuniq'.format(('_'.join(group_cols)),(counted)) if show_agg: print( "\nCounting unqiue ", counted, " by ", group_cols , '... and saved in', agg_name ) gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name}) df = df.merge(gp, on=group_cols, how='left') del gp gc.collect() if show_max: print( agg_name + " max value = ", df[agg_name].max() ) df[agg_name] = df[agg_name].astype(agg_type) # print('predictors',predictors) gc.collect() return( df ) # function to compute cumulative count def do_cumcount( df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True ): agg_name= '{}_by_{}_cumcount'.format(('_'.join(group_cols)),(counted)) if show_agg: print( "\nCumulative count by ", group_cols , '... and saved in', agg_name ) gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount() df[agg_name]=gp.values del gp gc.collect() if show_max: print( agg_name + " max value = ", df[agg_name].max() ) df[agg_name] = df[agg_name].astype(agg_type) # print('predictors',predictors) gc.collect() return( df ) # Calculating Variance by groups def do_var( df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True ): agg_name= '{}_by_{}_var'.format(('_'.join(group_cols)),(counted)) if show_agg: print( "\nCalculating variance of ", counted, " by ", group_cols , '... and saved in', agg_name ) gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name}) df = df.merge(gp, on=group_cols, how='left') del gp gc.collect() if show_max: print( agg_name + " max value = ", df[agg_name].max() ) df[agg_name] = df[agg_name].astype(agg_type) # print('predictors',predictors) gc.collect() return( df ) # Main cleansing function def preprocessing(chunk): chunk['hour'] = pd.to_datetime(chunk.click_time).dt.hour.astype('uint8'); gc.collect() chunk['day'] = pd.to_datetime(chunk.click_time).dt.day.astype('uint8'); gc.collect() chunk = do_next_Click( chunk,agg_suffix='nextClick', agg_type='float32' ); gc.collect() chunk = do_prev_Click( chunk,agg_suffix='prevClick', agg_type='float32' ); gc.collect() chunk = do_countuniq( chunk, ['ip'], 'channel' ); gc.collect() chunk = do_countuniq( chunk, ['ip', 'device', 'os'], 'app'); gc.collect() chunk = do_countuniq( chunk, ['ip', 'day'], 'hour' ); gc.collect() chunk = do_countuniq( chunk, ['ip'], 'app'); gc.collect() chunk = do_countuniq( chunk, ['ip', 'app'], 'os'); gc.collect() chunk = do_countuniq( chunk, ['ip'], 'device'); gc.collect() chunk = do_countuniq( chunk, ['app'], 'channel'); gc.collect() chunk = do_cumcount( chunk, ['ip'], 'os'); gc.collect() chunk = do_cumcount( chunk, ['ip', 'device', 'os'], 'app'); gc.collect() chunk = do_count( chunk, ['ip', 'day', 'hour'] ); gc.collect() chunk = do_count( chunk, ['ip', 'app']); gc.collect() chunk = do_count( chunk, ['ip', 'app', 'os']); gc.collect() chunk = do_var( chunk, ['ip', 'app', 'os'], 'hour'); gc.collect() del chunk['day'] ; gc.collect() del chunk['click_time'] ; gc.collect() del chunk['ip'] ; gc.collect return chunk def lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc', feval=None, early_stopping_rounds=20, num_boost_round=3000, verbose_eval=10, categorical_features=None, init_model=None, first_run = True, evals_results = None): lgb_params = { 'boosting_type': 'gbdt', 'objective': objective, 'metric':metrics, 'learning_rate': 0.04, #'is_unbalance': 'true', #because training data is unbalance (replaced with scale_pos_weight) 'num_leaves': 31, # we should let it be smaller than 2^(max_depth) 'max_depth': -1, # -1 means no limit 'min_child_samples': 20, # Minimum number of data need in a child(min_data_in_leaf) 'max_bin': 255, # Number of bucketed bin for feature values 'subsample': 0.6, # Subsample ratio of the training instance. 'subsample_freq': 0, # frequence of subsample, <=0 means no enable 'colsample_bytree': 0.3, # Subsample ratio of columns when constructing each tree. 'min_child_weight': 5, # Minimum sum of instance weight(hessian) needed in a child(leaf) 'subsample_for_bin': 200000, # Number of samples for constructing bin 'min_split_gain': 0, # lambda_l1, lambda_l2 and min_gain_to_split to regularization 'reg_alpha': 0.99, # L1 regularization term on weights 'reg_lambda': 0.9, # L2 regularization term on weights 'nthread': 8, 'verbose': 1, } lgb_params.update(params) print("preparing validation datasets \n") xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values, feature_name=predictors, categorical_feature=categorical_features ) xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values, feature_name=predictors, categorical_feature=categorical_features ) del dtrain del dvalid gc.collect() final_evals = {} bst1 = lgb.train(lgb_params, xgtrain, valid_sets=[xgtrain, xgvalid], valid_names=['train','valid'], evals_result=final_evals, num_boost_round=num_boost_round, early_stopping_rounds=early_stopping_rounds, verbose_eval=10, feval=feval, init_model=init_model, keep_training_booster = True) if first_run == False: for l in evals_results.keys(): for k in evals_results[l].keys(): evals_results[l][k].extend(final_evals[l][k]) else: evals_results = final_evals n_estimators = bst1.best_iteration print("\nModel Report") print("n_estimators : ", n_estimators) print(metrics+":", evals_results['valid'][metrics][n_estimators-1]) return bst1, evals_results def main(): # consider removing weak predictors such as second and day # initialize list of predictors predictors=['app', 'app_by_channel_countuniq', 'channel', 'device', 'hour', 'ip_app_by_os_countuniq', 'ip_app_device_os_channel_nextClick', 'ip_app_os_by_hour_var', 'ip_app_oscount', 'ip_appcount', 'ip_by_app_countuniq', 'ip_by_channel_countuniq', 'ip_by_device_countuniq', 'ip_by_os_cumcount', 'ip_channel_prevClick', 'ip_day_by_hour_countuniq', 'ip_day_hourcount', 'ip_device_os_by_app_countuniq', 'ip_device_os_by_app_cumcount', 'ip_os_device_app_nextClick', 'ip_os_device_nextClick', 'ip_os_prevClick', 'os'] categorical = ['app','device','os', 'channel', 'hour'] target = 'is_attributed' params = { 'learning_rate': 0.05, #'is_unbalance': 'true', # replaced with scale_pos_weight argument 'num_leaves': 15, # 2^max_depth - 1 'max_depth': 4, # -1 means no limit 'min_child_samples': 15, # Minimum number of data need in a child(min_data_in_leaf) 'max_bin': 200, # Number of bucketed bin for feature values 'subsample': 0.8, # Subsample ratio of the training instance. 'subsample_freq': 0, # frequence of subsample, <=0 means no enable 'colsample_bytree': 0.3, # Subsample ratio of columns when constructing each tree. 'min_child_weight': 5, # Minimum sum of instance weight(hessian) needed in a child(leaf) 'subsample_for_bin': 200000, # Number of samples for constructing bin 'min_split_gain': 0, # lambda_l1, lambda_l2 and min_gain_to_split to regularization 'reg_alpha': 0.99, # L1 regularization term on weights 'reg_lambda': 0.9, # L2 regularization term on weights 'scale_pos_weight':200 # because training data is extremely unbalanced } # USER DEFINED chunksize = 25000000 chunk_idx = 0 chunk_max = 4 split_ratio = 0.9 for chunk in pd.read_csv('../input/train.csv', chunksize=chunksize, parse_dates = ['click_time']): if chunk_idx >= chunk_max: print('Finished') del chunk gc.collect() break # Extracting new features print('processing chunk %d \n' % (chunk_idx + 1)) chunk = preprocessing(chunk) val_df = chunk[int(split_ratio*chunksize):chunksize] chunk = chunk[:int(split_ratio*chunksize)] if chunk_idx > 0: flag = False else: flag = True bst = None evals_results=None bst, evals_results = lgb_modelfit_nocv(params, chunk, val_df, predictors, target, objective='binary', metrics='auc', early_stopping_rounds=50, verbose_eval=True, num_boost_round=500, categorical_features=categorical, init_model=bst, first_run=flag, evals_results = evals_results) print("Processed chunk %d" % (chunk_idx + 1)) chunk_idx = chunk_idx + 1 del chunk gc.collect() test_df = pd.read_csv('../input/test.csv', parse_dates= ['click_time']) # current after preprocessing test_df = preprocessing(test_df) del val_df gc.collect() ax = lgb.plot_importance(bst, max_num_features=100) plt.show() plt.savefig('foo.png', dpi=200, bbox_inches='tight') sub = pd.DataFrame() sub['click_id'] = test_df['click_id'].astype('int') print("Predicting...") sub['is_attributed'] = bst.predict(test_df[predictors]) print("writing...") sub.to_csv('sub_lgb_balanced_new.csv',index=False) print("done...") del test_df gc.collect() if __name__ == '__main__': main()