import pandas as pd import numpy as np import datetime import gc import lightgbm as lgb from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score, roc_auc_score, confusion_matrix, roc_curve import os import matplotlib.pyplot as plt training_day_beginning = 144708152 training_day_end = 181878211 valid_day_beginning = 82259195 valid_day_end = valid_day_beginning + 10000000 #118735619 total_data_size_train = training_day_end - training_day_beginning + 1 total_data_size_valid = valid_day_end - valid_day_beginning + 1 total_data_size_submission = 18790469 reg_click_id_train_test_submission = total_data_size_train/total_data_size_submission def Data_Preparation(df): df['click_id'] = df.reset_index(drop=True).index df['click_time'] = pd.to_datetime(df.click_time) df['hour'] = df['click_time'].dt.hour.astype('uint8') df['minuteofday'] = df['click_time'].dt.hour *60 + df['click_time'].dt.minute gc.collect() print('grouping by ip-hour-app combination...') gp = df[['ip','hour', 'app', 'channel']].groupby(by=['ip','hour' , 'app' ])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_hour_count'}) df = df.merge(gp, on=['ip','hour', 'app'], how='left') del gp gc.collect() print('grouping by ip-app combination...') gp = df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'}) df = df.merge(gp, on=['ip','app'], how='left') del gp gc.collect() print('grouping by ip_channel combination') gp = df[['ip','hour','channel']].groupby(by=['ip','channel'])[['hour']].count().reset_index().rename(index=str, columns={'hour': 'ip_channel_count'}) df = df.merge(gp, on=['ip','channel'], how='left') del gp gc.collect() print('grouping by ip_app_os combination') gp = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].count().reset_index().rename(index=str, columns={'hour': 'ip_app_os_count'}) df = df.merge(gp, on=['ip','app', 'os'], how='left') del gp gc.collect() print('grouping by : ip_app_channel combination') gp = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].count().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_count'}) df = df.merge(gp, on=['ip','app', 'channel'], how='left') del gp gc.collect() print('looking for next clicks ...') df['nextClick'] = df.groupby(['ip', 'app', 'device', 'os']).click_time.shift(-1) df['nextClick'] = df.nextClick.fillna(np.datetime64('2017-11-15 00:00:00') ) df['DeltaNextClick'] = ((df['nextClick']- df.click_time) / datetime.timedelta(seconds=1)).apply(lambda x:int(np.round(x))) df = df.drop(['nextClick'] , axis = 1) gc.collect() print('looking for last clicks ...') df['lastClick'] = df.groupby(['ip', 'app', 'device', 'os']).click_time.shift(-1) df['lastClick'] = df.lastClick.fillna(np.datetime64('2018-11-01 00:00:00') ) df['DeltaLastClick'] = ((df['lastClick']- df.click_time) / datetime.timedelta(seconds=1)).apply(lambda x:int(np.round(x))) df = df.drop(['lastClick' , 'click_time' ] , axis = 1) gc.collect() df = df.fillna(0) print("vars and data type: ") df['app'] = df['app'].astype('uint16') df['channel'] = df['channel'].astype('uint16') df['ip_app_count'] = df['ip_app_count'].astype('uint16') df['ip_channel_count'] = df['ip_channel_count'].astype('uint16') df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint16') df['ip_app_channel_count'] = df['ip_app_channel_count'].astype('uint16') df['DeltaLastClick'] = df['DeltaLastClick'].astype('uint16') df['DeltaNextClick'] = df['DeltaNextClick'].astype('uint16') df.info() return df dtypes = { 'ip' : 'uint32', 'app' : 'uint16', 'device' : 'uint16', 'os' : 'uint16', 'channel' : 'uint16', 'is_attributed' : 'uint8', 'click_id' : 'uint32' } train_df = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', skiprows=range(1,training_day_beginning-1), nrows=training_day_end - training_day_beginning, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed']) gc.collect() train_df = Data_Preparation(train_df) train_df['is_attributed'] = train_df['is_attributed'].astype('uint16') gc.collect() valid_df = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', skiprows=range(1,valid_day_beginning-1), nrows=valid_day_end - valid_day_beginning, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed']) gc.collect() valid_df = Data_Preparation(valid_df) valid_df['is_attributed'] = valid_df['is_attributed'].astype('uint16') gc.collect() target = 'is_attributed' predictors = ['ip', 'app', 'device', 'os', 'channel', 'click_id', 'hour', 'minuteofday', 'ip_app_hour_count', 'ip_app_count', 'ip_channel_count', 'ip_app_os_count', 'ip_app_channel_count', 'DeltaNextClick', 'DeltaLastClick'] categorical = ['app', 'device', 'os', 'channel'] params = { 'boosting_type': 'gbdt', 'objective': 'binary', 'metric':'auc', 'num_leaves': 31, # we should let it be smaller than 2^(max_depth) 'max_depth': 7, # -1 means no limit 'min_child_samples': 20, # Minimum number of data need in a child(min_data_in_leaf) 'learning_rate': 0.2, 'subsample': 0.6, # Subsample ratio of the training instance. 'n_estimators' : 153, 'max_bin': 255, # Number of bucketed bin for feature values 'subsample_freq': 0, # frequence of subsample, <=0 means no enable 'colsample_bytree': 0.3, # Subsample ratio of columns when constructing each tree. 'min_child_weight': 1, # Minimum sum of instance weight(hessian) needed in a child(leaf) 'subsample_for_bin': 200000, # Number of samples for constructing bin 'min_split_gain': 0, # lambda_l1, lambda_l2 and min_gain_to_split to regularization 'reg_alpha': 0, # L1 regularization term on weights 'reg_lambda': 0, # L2 regularization term on weights 'nthread': 4, 'verbose': 0, 'scale_pos_weight' : 409, } gc.collect() print("Training...") dtrain = lgb.Dataset(train_df[predictors].values, label=train_df[target].values, feature_name=predictors, categorical_feature=categorical ) dvalid = lgb.Dataset(valid_df[predictors].values, label=valid_df[target].values, feature_name=predictors, categorical_feature=categorical ) gc.collect() print("Training the model...") evals_result = {} lgb_model = lgb.train( params, dtrain, valid_sets=[dtrain, dvalid], valid_names=['train','valid'], evals_result = evals_result, num_boost_round=1000, early_stopping_rounds=20, verbose_eval=50, feval=None ) valid_target = valid_df['is_attributed'] valid_pred = lgb_model.predict(valid_df[predictors]) print ("LGBM : \n" , "accuracy: " , accuracy_score(valid_target , np.round(valid_pred) ) , "\n" , "recall: " , recall_score(valid_target , np.round(valid_pred)), "\n" , "precision: " , precision_score(valid_target , np.round(valid_pred)), "\n" , "f1: ", f1_score(valid_target , np.round(valid_pred)) , "\n" , "roc_auc_score: " , roc_auc_score(valid_target , np.round(valid_pred)), "\n" "confusion_matrix :" , "\n" , confusion_matrix(valid_target , np.round(valid_pred)) ) fpr, tpr, _ = roc_curve(valid_target, valid_pred) fig = plt.figure() curve = fig.add_subplot(111) curve.plot(fpr,tpr,label="Courbe ROC du modèle" , color='red') curve.set(xlabel = 'Taux de faux positifs' , ylabel ='Taux de vrais positifs') random_line = fig.add_subplot(111) random_line.plot([0,1] , [1,1], label = 'Courbe ROC pour prédictions idéales', color = 'black') fig.legend(loc=4) plt.show() import matplotlib.pyplot as plt f, ax = plt.subplots(figsize=[7,10]) lgb.plot_importance(lgb_model, ax=ax, max_num_features=len(predictors)) plt.title("Light GBM Feature Importance") plt.savefig('feature_import.png') del valid_df, train_df ,dtrain, dvalid gc.collect() #****************************************************************** #********************Submission********************** #****************************************************************** test_df = pd.read_csv("../input/talkingdata-adtracking-fraud-detection/test.csv", dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id']) submission_df = pd.DataFrame(columns=["is_attributed"], index = test_df['click_id']) submission_df.index.names = ['click_id'] test_df = Data_Preparation(test_df) submission_df["is_attributed"] = lgb_model.predict(test_df[predictors]) submission_df.to_csv('submission.csv')