{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false,"collapsed":true},"cell_type":"code","source":"#Our final submission is a single lightgbm model, no blending (due to limited computational power and time)\n#We are going topresent all the pre_processing, feature engineering functions and lightgbm paramaters, enjoy","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\n#import pytz\nimport gc\nfrom sklearn import preprocessing","execution_count":1,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4c2805ccd9c3ca5fa845531893dc07f2285c4882"},"cell_type":"code","source":"#all functions\ndef prepare(df):\n    df['row_id'] = range(df.shape[0])\n    df.click_time = pd.to_datetime(df.click_time)\n    df['day'] = df.click_time.dt.day.astype('uint8')\n    df['hour'] = df.click_time.dt.hour.astype('uint8')\n    \n    most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]\n    least_freq_hours_in_test_data = [6, 11, 15]\n    \n    df['in_test_hh'] = (   3 \n                         - 2*df['hour'].isin(  most_freq_hours_in_test_data ) \n                         - 1*df['hour'].isin( least_freq_hours_in_test_data ) ).astype('uint8')\n    return df\n\ndef get_size(group_column, df):\n    '''\n    group_column: categorical columns to group on \n    '''\n    \n    used_cols = group_column.copy()\n    used_cols.extend(['row_id'])\n    all_df = df[used_cols]\n    grouped = all_df[group_column].groupby(group_column)\n    \n    #size of each level in group_column\n    the_size = pd.DataFrame(grouped.size()).reset_index()\n    names = group_column.copy()\n    new_name = \"_\".join(x for x in names) + '_size'\n    names.append(new_name)\n    the_size.columns = names\n    \n    all_df = pd.merge(all_df, the_size)\n    all_df.sort_values('row_id', inplace=True)\n\n    df[new_name] = np.array(all_df[new_name])\n    del all_df\n    gc.collect()\n    \n    return df\n\ndef get_unique(df, grouping_col, target_col):\n    \n    used_cols = grouping_col.copy()\n    used_cols.extend(['row_id'])\n    used_cols.extend(target_col)\n    all_df = df[used_cols]\n    \n    group_used_cols = grouping_col.copy()\n    group_used_cols.extend(target_col)\n    grouped = all_df[group_used_cols].groupby(grouping_col)\n    #unique count\n    the_count = pd.DataFrame(grouped[target_col].nunique()).reset_index()\n    names = grouping_col.copy()\n    new_name = \"_\".join(x for x in target_col) + '_unique_count_on_' + \"_\".join(x for x in grouping_col)\n    names.append(new_name)\n    the_count.columns = names\n    \n    all_df = pd.merge(all_df, the_count)\n    all_df.sort_values('row_id', inplace=True)\n\n    df[new_name] = np.array(all_df[new_name])\n    del all_df\n    gc.collect()\n    \n    return df\n\ndef get_user_info(df):\n    user = ['ip', 'device','os']\n    #get total count\n    df = get_size(user, df)\n    new_name1 = \"_\".join(x for x in user) + '_size'\n    #get total count on an app\n    df = get_size(['ip', 'app', 'device','os'], df)\n    new_name2 = \"_\".join(x for x in (['ip', 'app', 'device','os'])) + '_size'\n    #get proportion of app count on this user\n    new_name3 = new_name2 + '/' + new_name1\n    df[new_name3] = df[new_name2]/df[new_name1]\n    #get unique app count\n    df = get_unique(df, user, ['app'])\n    new_name4 = \"_\".join(x for x in ['app']) + '_unique_count_on_' + \"_\".join(x for x in user)\n    #get unique/size ratio\n    new_name5 = new_name4 + '/' + new_name1\n    df[new_name5] = df[new_name4]/df[new_name1]\n    #cumcount\n    df['user_newness'] = df.groupby(user).cumcount() \n    df['user_app_oldness'] = df.groupby(user + ['app']).cumcount(ascending=False)\n    df.drop(['ip_device_os_size'], axis=1, inplace=True)\n    gc.collect()\n    print('get_user_info done')\n    return df\n\ndef get_kernel_fe(df):\n    df = get_size(['ip','day','in_test_hh'], df)\n    df = get_size(['ip','day','hour'], df)\n    #df = get_size(['ip','day','hour','os'], df)\n    df = get_size(['ip','day','hour','app'], df)\n    df = get_size(['ip','day','hour','app','os'], df)\n    print('get_kernel_fe half is done')\n    df = get_size(['app','day','hour'], df)\n    df = get_size(['ip','app'], df)\n    df = get_size(['ip','app','os'], df)\n    df = get_size(['ip','device'], df)\n    df = get_size(['app','channel'], df)\n    print('get_kernel_fe is done')\n    gc.collect()\n    return df    \n\ndef get_hourly_app_info(df):\n    app = ['app', 'day','hour']\n    df = get_unique(df, app, ['ip'])\n    new_name1 = \"_\".join(x for x in ['ip']) + '_unique_count_on_' + \"_\".join(x for x in app)\n    new_name2 = \"_\".join(x for x in app) + '_size'\n    new_name3 = new_name1 + '/' + new_name2\n    df[new_name3] = df[new_name1]/df[new_name2]\n    \n    df = get_size(['ip', 'app','device', 'os', 'day','hour'], df)\n    gc.collect()\n    print('get_hourly_app_info done')\n    return df\n\ndef get_click_time_info(df):\n    df.click_time = pd.to_datetime(df.click_time, errors = 'ignore')\n    used_cols = ['ip', 'app', 'device', 'os', 'click_time', 'row_id']\n    all_df = df[used_cols]\n    all_df = all_df.sort_values(by=used_cols)\n    \n    all_df['next_ip']=all_df.ip.shift(-1)\n    all_df['next_app']=all_df.app.shift(-1)\n    all_df['next_device']=all_df.device.shift(-1)\n    all_df['next_os']=all_df.os.shift(-1)\n    all_df['next_click_time']=all_df.click_time.shift(-1)\n    \n    all_df['has_next_ip'] = np.where(all_df.ip == all_df.next_ip, 1, 0)\n    all_df['has_next_app'] = np.where(all_df.app == all_df.next_app, 1, 0)\n    all_df['has_next_device'] = np.where(all_df.device == all_df.next_device, 1, 0)\n    all_df['has_next_os'] = np.where(all_df.os == all_df.next_os, 1, 0)\n    \n    all_df['next_click'] = np.where((all_df.has_next_ip == 1) & (all_df.has_next_app == 1) &(all_df.has_next_device == 1) & (all_df.has_next_os == 1) , (all_df.next_click_time - all_df.click_time)/np.timedelta64(1, 's'), np.NaN)\n    \n    \n    all_df['previous_ip']=all_df.ip.shift(1)\n    all_df['previous_app']=all_df.app.shift(1)\n    all_df['previous_device']=all_df.device.shift(1)\n    all_df['previous_os']=all_df.os.shift(1)\n    all_df['previous_click_time']=all_df.click_time.shift(1)\n    \n    all_df['has_previous_ip'] = np.where(all_df.ip == all_df.previous_ip, 1, 0)\n    all_df['has_previous_app'] = np.where(all_df.app == all_df.previous_app, 1, 0)\n    all_df['has_previous_device'] = np.where(all_df.device == all_df.previous_device, 1, 0)\n    all_df['has_previous_os'] = np.where(all_df.os == all_df.previous_os, 1, 0)\n    \n    all_df['last_click'] = np.where((all_df.has_previous_ip == 1) & (all_df.has_previous_app == 1) & (all_df.has_previous_device == 1) & (all_df.has_previous_os == 1) , (all_df.click_time-all_df.previous_click_time)/np.timedelta64(1, 's'), np.NaN)\n    \n    all_df = all_df.sort_values(by=['row_id'])\n    df['next_click'] = np.array(all_df['next_click'])\n    df['last_click'] = np.array(all_df['last_click'])\n    del all_df\n    gc.collect()\n    print('get_click_time_info done')\n    return df\n\ndef get_next_click_stat(df):\n    grouping_col = ['ip','app','device','os']\n    target_col = ['next_click']\n    \n    used_cols = grouping_col.copy()\n    used_cols.extend(['row_id'])\n    used_cols.extend(target_col)\n    all_df = df[used_cols]\n    \n    group_used_cols = grouping_col.copy()\n    group_used_cols.extend(target_col)\n    grouped = all_df[group_used_cols].groupby(grouping_col)\n    \n    new_names = []\n    #mean\n    the_mean = pd.DataFrame(grouped[target_col].mean()).reset_index()\n    names = grouping_col.copy()\n    new_name = 'next_click_mean'\n    new_names.append(new_name)\n    names.append(new_name)\n    the_mean.columns = names\n    #median\n    the_median = pd.DataFrame(grouped[target_col].median()).reset_index()\n    names = grouping_col.copy()\n    new_name = 'next_click_median'\n    new_names.append(new_name)\n    names.append(new_name)\n    the_median.columns = names\n    the_stats = pd.merge(the_mean, the_median)\n    #max\n    the_max = pd.DataFrame(grouped[target_col].max()).reset_index()\n    names = grouping_col.copy()\n    new_name = 'next_click_max'\n    new_names.append(new_name)\n    names.append(new_name)\n    the_max.columns = names\n    the_stats = pd.merge(the_stats, the_max)\n    \n    all_df = pd.merge(all_df, the_stats)\n    all_df.sort_values('row_id', inplace=True)\n    \n    for new_name in new_names:\n        df[new_name] = np.array(all_df[new_name])\n    del all_df\n    gc.collect()\n    print('get_next_click_stat is done')\n    return df\n\ndef get_old_size(df):\n    candidates = [        \n        ['ip', 'app', 'device','os'],\n        ['app', 'device','os']\n    ]\n    df['minute'] = df.click_time.dt.minute.astype('uint8')\n    df['second'] = df.click_time.dt.second.astype('uint8')\n    gc.collect()\n    for i in range(0, 2):\n        used = candidates[i].copy()\n        if i == 0:\n            df = get_size(used + ['day', 'hour', 'minute'], df)\n            df = get_size(used + ['day', 'hour', 'minute', 'second'], df)\n            df['ip_app_device_os_size_hour/min_rate'] = df['ip_app_device_os_day_hour_size']/df['ip_app_device_os_day_hour_minute_size']\n            df['ip_app_device_os_size_min/sec_rate'] = df['ip_app_device_os_day_hour_minute_size']/df['ip_app_device_os_day_hour_minute_second_size']\n            dropped = ['ip_app_device_os_day_hour_minute_second_size']\n            df.drop(dropped, axis=1, inplace=True)\n            gc.collect()\n        elif i == 1:\n            df = get_size(used + ['day', 'hour'], df)\n            df = get_size(used + ['day', 'hour', 'minute'], df)\n            df = get_size(used + ['day', 'hour', 'minute', 'second'], df)\n            df['app_device_os_size_hour/min_rate'] = df['app_device_os_day_hour_size']/df['app_device_os_day_hour_minute_size']\n            df['app_device_os_size_hour/sec_rate'] = df['app_device_os_day_hour_size']/df['app_device_os_day_hour_minute_second_size']\n            df['app_device_os_size_min/sec_rate'] = df['app_device_os_day_hour_minute_size']/df['app_device_os_day_hour_minute_second_size']\n            dropped = ['app_device_os_day_hour_minute_size', 'app_device_os_day_hour_minute_second_size']\n            df.drop(dropped, axis=1, inplace=True)\n            gc.collect()\n    df.drop(['minute','second'], axis=1, inplace=True)\n    print('get old size done')\n    gc.collect()\n    return df\n\ndef get_old_unique(df):\n    #os\n    df = get_size(['ip'],df)\n    df = get_unique(df, ['ip'], ['os'])\n    new_name1 = 'os_unique_count_on_ip' + '/' + 'ip_size'\n    df[new_name1] = df['os_unique_count_on_ip']/df['ip_size']\n    #device\n    df = get_unique(df, ['ip'], ['device'])\n    new_name2 = 'device_unique_count_on_ip' + '/' + 'ip_size'\n    df[new_name2] = df['device_unique_count_on_ip']/df['ip_size']\n    \n    df.drop(['ip_size'], axis=1, inplace=True)\n    gc.collect()\n    print('get old unique done')\n    return df\n\ndef get_yulia_fe(df):\n    df = get_unique(df, ['ip','device','os','day','hour'], ['channel'])\n    df = get_size(['ip','channel'],df)\n    print('get yulia fe is done')\n    return df\n\ndef get_p1(train, num_rounds, if_ip=False):\n    predictors = list(train.columns)\n    remove_list = ['click_id', 'row_id', 'day', 'minute', 'second', 'click_time', 'local_click_time', 'attributed_time', 'is_attributed']\n    for ele in remove_list:\n        if ele in predictors:\n            predictors.remove(ele)\n    target = 'is_attributed'\n    categorical = ['ip','app','os','device','channel','hour']\n    if if_ip == False:\n        predictors.remove('ip')\n        categorical.remove('ip')\n    params = {\n        'boosting_type': 'gbdt', 'objective': 'binary', 'nthread': -1, 'silent': True, 'metric':'auc', 'seed':77,\n        'num_leaves': 48, 'learning_rate': 0.01, 'max_depth': -1, 'gamma':47,\n        'max_bin': 255, 'subsample_for_bin': 70000, 'bagging_fraction':0.7, 'bagging_freq':1, 'bagging_seed':55,\n        'colsample_bytree': 0.6548, 'reg_alpha': 19.43, 'reg_lambda': 0, \n        'min_split_gain': 0.3512, 'min_child_weight': 0, 'min_child_samples':1321, 'scale_pos_weight':205}\n    \n    \n    xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical\n                          )\n    bst = lgb.train(params, xgtrain, num_boost_round = num_rounds, verbose_eval=False)\n    return bst, predictors","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5264a3feaa07ad8dc654f7197e396016cd4989ff"},"cell_type":"code","source":"#the code you need to run to get the model\n'''\ntrain = pd.read_csv('your_path/train.csv', dtype = dtypes)\nold_test = pd.read_csv('your_path/test_supplement.csv', dtype = dtypes)\ntrain = pd.concat([train,old_test], axis=0, ignore_index=True)\ndel old_test\ngc.collect()\n#pre_processing\nprepare(train)\ntrain = get_user_info(train)\ntrain = get_kernel_fe(train)\ntrain = get_hourly_app_info(train)\ntrain = get_click_time_info(train)\ntrain = get_next_click_stat(train)\ntrain = get_old_size(train)\ntrain = get_old_unique(train)\ntrain = get_yulia_fe(train)\ntrain = train.sort_values(by=['row_id'])\ngc.collect()\n#split\ntrain, old_test = train.iloc[:184903890,:], train.iloc[184903890:,:]\ntrain.drop(['click_id'], axis=1, inplace=True)\nold_test.drop(['attributed_time', 'is_attributed'], axis=1, inplace=True)\nold_test.rename(columns={'click_id': 'old_click_id'}, inplace=True)\n#get model and predict(so you could do things such as model.predict(your_test[predictors]))\nmodel, predictors = get_p1(train, 2200, if_ip=False)\n'''\n#then use your own way to map from test_supplement to test and use the returned model and predictors to get the prediction, that's all ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}