{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ncolumns_required = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\npath= '~/.kaggle/competitions/talkingdata-adtracking-fraud-detection/'\n# Load subset of the training data\nX_train = pd.read_csv('../input/train.csv',nrows=100000,usecols=columns_required,parse_dates=['click_time'])\n\n# Show the head of the table\nX_train.head()\nX_train.shape","execution_count":1,"outputs":[{"output_type":"execute_result","execution_count":1,"data":{"text/plain":"(100000, 7)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# segregating the click_time column into day, hour, minute and second","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train['day'] = X_train['click_time'].dt.day.astype('uint8')\nX_train['hour'] = X_train['click_time'].dt.hour.astype('uint8')\nX_train['minute'] = X_train['click_time'].dt.minute.astype('uint8')\nX_train['second'] = X_train['click_time'].dt.second.astype('uint8')\nX_train.head()","execution_count":3,"outputs":[{"output_type":"execute_result","execution_count":3,"data":{"text/plain":" ip app device os ... day hour minute second\n0 83230 3 1 13 ... 6 14 32 21\n1 17357 3 1 19 ... 6 14 33 34\n2 35810 3 1 13 ... 6 14 34 12\n3 45745 14 1 13 ... 6 14 34 52\n4 161007 3 1 13 ... 6 14 35 8\n\n[5 rows x 11 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timeis_attributeddayhourminutesecond
08323031133792017-11-06 14:32:2106143221
11735731193792017-11-06 14:33:3406143334
23581031133792017-11-06 14:34:1206143412
345745141134782017-11-06 14:34:5206143452
416100731133792017-11-06 14:35:080614358
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.loc[:, 'device'] = X_train.loc[:, 'device'].astype(np.int16)\nX_train.loc[:, 'os'] = X_train.loc[:, 'os'].astype(np.int16)\nX_train.loc[:, 'channel'] = X_train.loc[:, 'channel'].astype(np.int16)\nX_train.loc[:, 'is_attributed'] = X_train.loc[:, 'is_attributed'].astype(np.int8)\nX_train.loc[:, 'ip'] = X_train.loc[:, 'ip'].astype(np.int8)\nX_train.loc[:, 'app'] = X_train.loc[:, 'app'].astype(np.int8)\n","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"(100000, 11)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns_required = ['ip', 'app', 'device', 'os', 'channel', 'click_time']\n\n# Load subset of the training data\nX_test = pd.read_csv('../input/test.csv',nrows=100000,usecols=columns_required,parse_dates=['click_time'])\nX_test.fillna(X_test.mean(), inplace=True)\n# Show the head of the table\nX_test.head()\nX_test.shape","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"(100000, 6)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test['day'] = X_test['click_time'].dt.day.astype('uint8')\nX_test['hour'] = X_test['click_time'].dt.hour.astype('uint8')\nX_test['minute'] = X_test['click_time'].dt.minute.astype('uint8')\nX_test['second'] = X_test['click_time'].dt.second.astype('uint8')\nX_test.head()","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":" ip app device os ... day hour minute second\n0 5744 9 1 3 ... 10 4 0 0\n1 119901 9 1 3 ... 10 4 0 0\n2 72287 21 1 19 ... 10 4 0 0\n3 78477 15 1 13 ... 10 4 0 0\n4 123080 12 1 13 ... 10 4 0 0\n\n[5 rows x 10 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timedayhourminutesecond
057449131072017-11-10 04:00:0010400
11199019134662017-11-10 04:00:0010400
272287211191282017-11-10 04:00:0010400
378477151131112017-11-10 04:00:0010400
4123080121133282017-11-10 04:00:0010400
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.loc[:, 'device'] = X_test.loc[:, 'device'].astype(np.int16)\nX_test.loc[:, 'os'] = X_test.loc[:, 'os'].astype(np.int16)\nX_test.loc[:, 'channel'] = X_test.loc[:, 'channel'].astype(np.int16)\nX_test.loc[:, 'ip'] = X_test.loc[:, 'ip'].astype(np.int8)\nX_test.loc[:, 'app'] = X_test.loc[:, 'app'].astype(np.int8)\n","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ATTRIBUTION_CATEGORIES = [ \n # Group-1 Features \n ['ip'], ['app'], ['device'], ['os'], ['channel'],\n \n # Group-2 Features\n ['app', 'channel'],\n ['app', 'os'],\n ['app', 'device'],\n \n # Group-3 Features\n ['channel', 'os'],\n ['channel', 'device'],\n ['os', 'device']\n]\n","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find frequency of is_attributed for each unique value in column in train data\nfreqs = {}\nfor cols in ATTRIBUTION_CATEGORIES:\n \n # New feature name\n new_feature = '_'.join(cols)+'_confRate' \n \n # Perform the groupby\n group_object = X_train.groupby(cols)\n \n # Group sizes \n group_sizes = group_object.size()\n log_group = np.log(100000) \n print(\">> Calculating confidence-weighted rate for: {}.\\n Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}\".format(\n cols, new_feature, \n group_sizes.max(), \n np.round(group_sizes.mean(), 2),\n np.round(group_sizes.median(), 2),\n group_sizes.min()\n ))\n \n # Aggregation function\n def rate_calculation(x):\n \"\"\"Calculate the attributed rate. Scale by confidence\"\"\"\n rate = x.sum() / float(x.count())\n conf = np.min([1, np.log(x.count()) / log_group])\n return rate * conf\n \n # Merge operation\n X_train = X_train.merge(\n group_object['is_attributed']. \\\n apply(rate_calculation). \\\n reset_index(). \\\n rename( \n index=str,\n columns={'is_attributed': new_feature}\n )[cols + [new_feature]],\n on=cols, how='left'\n )\n \nX_train.head()","execution_count":10,"outputs":[{"output_type":"stream","text":">> Calculating confidence-weighted rate for: ['ip'].\n Saving to: ip_confRate. Group Max /Mean / Median / Min: 1059 / 390.62 / 375.0 / 205\n>> Calculating confidence-weighted rate for: ['app'].\n Saving to: app_confRate. Group Max /Mean / Median / Min: 13280 / 909.09 / 6.0 / 1\n>> Calculating confidence-weighted rate for: ['device'].\n Saving to: device_confRate. Group Max /Mean / Median / Min: 94397 / 1694.92 / 1.0 / 1\n>> Calculating confidence-weighted rate for: ['os'].\n Saving to: os_confRate. Group Max /Mean / Median / Min: 23957 / 1052.63 / 67.0 / 1\n>> Calculating confidence-weighted rate for: ['channel'].\n Saving to: channel_confRate. Group Max /Mean / Median / Min: 10582 / 751.88 / 188.0 / 1\n>> Calculating confidence-weighted rate for: ['app', 'channel'].\n Saving to: app_channel_confRate. Group Max /Mean / Median / Min: 10015 / 316.46 / 24.5 / 1\n>> Calculating confidence-weighted rate for: ['app', 'os'].\n Saving to: app_os_confRate. Group Max /Mean / Median / Min: 3236 / 66.49 / 6.0 / 1\n>> Calculating confidence-weighted rate for: ['app', 'device'].\n Saving to: app_device_confRate. Group Max /Mean / Median / Min: 12455 / 478.47 / 2.0 / 1\n>> Calculating confidence-weighted rate for: ['channel', 'os'].\n Saving to: channel_os_confRate. Group Max /Mean / Median / Min: 2540 / 28.66 / 5.0 / 1\n>> Calculating confidence-weighted rate for: ['channel', 'device'].\n Saving to: channel_device_confRate. Group Max /Mean / Median / Min: 10250 / 342.47 / 7.0 / 1\n>> Calculating confidence-weighted rate for: ['os', 'device'].\n Saving to: os_device_confRate. Group Max /Mean / Median / Min: 23217 / 462.96 / 8.0 / 1\n","name":"stdout"},{"output_type":"execute_result","execution_count":10,"data":{"text/plain":" ip app ... channel_device_confRate os_device_confRate\n0 30 3 ... 0.0 0.001187\n1 -51 3 ... 0.0 0.001429\n2 -30 3 ... 0.0 0.001187\n3 -79 14 ... 0.0 0.001187\n4 -17 3 ... 0.0 0.001187\n\n[5 rows x 22 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timeis_attributeddayhourminutesecondip_confRateapp_confRatedevice_confRateos_confRatechannel_confRateapp_channel_confRateapp_os_confRateapp_device_confRatechannel_os_confRatechannel_device_confRateos_device_confRate
03031133792017-11-06 14:32:21061432210.0013060.0002960.0013070.0011490.00.00.000270.0003060.00.00.001187
1-5131193792017-11-06 14:33:34061433340.0000000.0002960.0013070.0013890.00.00.000000.0003060.00.00.001429
2-3031133792017-11-06 14:34:12061434120.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.001187
3-79141134782017-11-06 14:34:52061434520.0006710.0006850.0013070.0011490.00.00.000000.0006980.00.00.001187
4-1731133792017-11-06 14:35:0806143580.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.001187
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Test Data\nfreqs = {}\nfor cols in ATTRIBUTION_CATEGORIES:\n \n # New feature name\n new_feature = '_'.join(cols)+'_confRate' \n \n # Perform the groupby\n group_object = X_test.groupby(cols)\n \n # Group sizes \n group_sizes = group_object.size()\n log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence \n print(\">> Calculating confidence-weighted rate for: {}.\\n Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}\".format(\n cols, new_feature, \n group_sizes.max(), \n np.round(group_sizes.mean(), 2),\n np.round(group_sizes.median(), 2),\n group_sizes.min()\n ))\n","execution_count":11,"outputs":[{"output_type":"stream","text":">> Calculating confidence-weighted rate for: ['ip'].\n Saving to: ip_confRate. Group Max /Mean / Median / Min: 839 / 390.62 / 380.0 / 222\n>> Calculating confidence-weighted rate for: ['app'].\n Saving to: app_confRate. Group Max /Mean / Median / Min: 15358 / 787.4 / 4.0 / 1\n>> Calculating confidence-weighted rate for: ['device'].\n Saving to: device_confRate. Group Max /Mean / Median / Min: 95400 / 1041.67 / 1.0 / 1\n>> Calculating confidence-weighted rate for: ['os'].\n Saving to: os_confRate. Group Max /Mean / Median / Min: 24304 / 952.38 / 65.0 / 1\n>> Calculating confidence-weighted rate for: ['channel'].\n Saving to: channel_confRate. Group Max /Mean / Median / Min: 6042 / 689.66 / 272.0 / 1\n>> Calculating confidence-weighted rate for: ['app', 'channel'].\n Saving to: app_channel_confRate. Group Max /Mean / Median / Min: 4746 / 265.25 / 12.0 / 1\n>> Calculating confidence-weighted rate for: ['app', 'os'].\n Saving to: app_os_confRate. Group Max /Mean / Median / Min: 3716 / 60.17 / 6.0 / 1\n>> Calculating confidence-weighted rate for: ['app', 'device'].\n Saving to: app_device_confRate. Group Max /Mean / Median / Min: 14303 / 374.53 / 2.0 / 1\n>> Calculating confidence-weighted rate for: ['channel', 'os'].\n Saving to: channel_os_confRate. Group Max /Mean / Median / Min: 1129 / 24.48 / 5.0 / 1\n>> Calculating confidence-weighted rate for: ['channel', 'device'].\n Saving to: channel_device_confRate. Group Max /Mean / Median / Min: 5950 / 262.47 / 4.0 / 1\n>> Calculating confidence-weighted rate for: ['os', 'device'].\n Saving to: os_device_confRate. Group Max /Mean / Median / Min: 23963 / 387.6 / 6.0 / 1\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define all the groupby transformations\nGROUPBY_AGGREGATIONS = [\n \n # Group-1 - GroupBy Features \n # Variance in day, for ip-app-channel\n {'groupby': ['ip','app','channel'], 'select': 'day', 'agg': 'var'},\n # Variance in hour, for ip-app-os\n {'groupby': ['ip','app','os'], 'select': 'hour', 'agg': 'var'},\n # Variance in hour, for ip-day-channel\n {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'},\n # Count, for ip-day-hour\n {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},\n # Count, for ip-app\n {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'}, \n # Count, for ip-app-os\n {'groupby': ['ip', 'app', 'os'], 'select': 'channel', 'agg': 'count'},\n # Count, for ip-app-day-hour\n {'groupby': ['ip','app','day','hour'], 'select': 'channel', 'agg': 'count'},\n # Mean hour, for ip-app-channel\n {'groupby': ['ip','app','channel'], 'select': 'hour', 'agg': 'mean'}, \n \n # Group-2 - GroupBy Features \n # Average clicks on app by distinct users; is it an app they return to?\n {'groupby': ['app'], \n 'select': 'ip', \n 'agg': lambda x: float(len(x)) / len(x.unique()), \n 'agg_name': 'AvgViewPerDistinct'\n },\n # How popular is the app or channel?\n {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},\n {'groupby': ['channel'], 'select': 'app', 'agg': 'count'},\n \n # Group-3 - GroupBy Features \n # Reference from https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977 \n {'groupby': ['ip'], 'select': 'channel', 'agg': 'nunique'}, \n {'groupby': ['ip'], 'select': 'app', 'agg': 'nunique'}, \n {'groupby': ['ip','day'], 'select': 'hour', 'agg': 'nunique'}, \n {'groupby': ['ip','app'], 'select': 'os', 'agg': 'nunique'}, \n {'groupby': ['ip'], 'select': 'device', 'agg': 'nunique'}, \n {'groupby': ['app'], 'select': 'channel', 'agg': 'nunique'}, \n {'groupby': ['ip', 'device', 'os'], 'select': 'app', 'agg': 'nunique'}, \n {'groupby': ['ip','device','os'], 'select': 'app', 'agg': 'cumcount'}, \n {'groupby': ['ip'], 'select': 'app', 'agg': 'cumcount'}, \n {'groupby': ['ip'], 'select': 'os', 'agg': 'cumcount'}, \n {'groupby': ['ip','day','channel'], 'select': 'hour', 'agg': 'var'} \n \n]","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n \n # Name of the aggregation we're applying\n agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n \n # Name of new feature\n new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])\n \n # Info\n print(\"Grouping by {}, and aggregating {} with {}\".format(\n spec['groupby'], spec['select'], agg_name\n ))\n \n # Unique list of features to select\n all_features = list(set(spec['groupby'] + [spec['select']]))\n # Perform the groupby\n gp = X_train[all_features]. \\\n groupby(spec['groupby'])[spec['select']]. \\\n agg(spec['agg']). \\\n reset_index(). \\\n rename(index=str, columns={spec['select']: new_feature})\n \n # Merge back to X_total\n if 'cumcount' == spec['agg']:\n X_train[new_feature] = gp[0].values\n else:\n X_train = X_train.merge(gp, on=spec['groupby'], how='left')\n \n # Clear memory\n del gp\n gc.collect()\n\nX_train.head()","execution_count":13,"outputs":[{"output_type":"stream","text":"Grouping by ['ip', 'app', 'channel'], and aggregating day with var\nGrouping by ['ip', 'app', 'os'], and aggregating hour with var\nGrouping by ['ip', 'day', 'channel'], and aggregating hour with var\nGrouping by ['ip', 'day', 'hour'], and aggregating channel with count\nGrouping by ['ip', 'app'], and aggregating channel with count\nGrouping by ['ip', 'app', 'os'], and aggregating channel with count\nGrouping by ['ip', 'app', 'day', 'hour'], and aggregating channel with count\nGrouping by ['ip', 'app', 'channel'], and aggregating hour with mean\nGrouping by ['app'], and aggregating ip with AvgViewPerDistinct\nGrouping by ['app'], and aggregating channel with count\nGrouping by ['channel'], and aggregating app with count\nGrouping by ['ip'], and aggregating channel with nunique\nGrouping by ['ip'], and aggregating app with nunique\nGrouping by ['ip', 'day'], and aggregating hour with nunique\nGrouping by ['ip', 'app'], and aggregating os with nunique\nGrouping by ['ip'], and aggregating device with nunique\nGrouping by ['app'], and aggregating channel with nunique\nGrouping by ['ip', 'device', 'os'], and aggregating app with nunique\nGrouping by ['ip', 'device', 'os'], and aggregating app with cumcount\nGrouping by ['ip'], and aggregating app with cumcount\nGrouping by ['ip'], and aggregating os with cumcount\nGrouping by ['ip', 'day', 'channel'], and aggregating hour with var\n","name":"stdout"},{"output_type":"execute_result","execution_count":13,"data":{"text/plain":" ip app ... ip_cumcount_os ip_day_channel_var_hour_y\n0 30 3 ... 0 0.700000\n1 -51 3 ... 0 0.500000\n2 -30 3 ... 0 0.916667\n3 -79 14 ... 0 1.333333\n4 -17 3 ... 0 0.571429\n\n[5 rows x 44 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timeis_attributeddayhourminutesecondip_confRateapp_confRatedevice_confRateos_confRatechannel_confRateapp_channel_confRateapp_os_confRateapp_device_confRatechannel_os_confRatechannel_device_confRateos_device_confRateip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_y
03031133792017-11-06 14:32:21061432210.0013060.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.3589740.70000024013214.842.6406251091613536928313429140000.700000
1-5131193792017-11-06 14:33:34061433340.0000000.0002960.0013070.0013890.00.00.000000.0003060.00.00.0014290.00.2666670.50000015515115.542.6406251091613536823314429160000.500000
2-3031133792017-11-06 14:34:12061434120.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.5714290.9166671287114.542.6406251091613536025313329150000.916667
3-79141134782017-11-06 14:34:52061434520.0006710.0006850.0013070.0011490.00.00.000000.0006980.00.00.001187NaN0.4444441.3333331319114.011.92578130531267431311519210001.333333
4-1731133792017-11-06 14:35:0806143580.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.2500000.57142916216115.042.6406251091613536928220429190000.571429
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Test Data \n#Apply all the groupby transformations\nfor spec in GROUPBY_AGGREGATIONS:\n \n # Name of the aggregation we're applying\n agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n \n # Name of new feature\n new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])\n \n # Info\n print(\"Grouping by {}, and aggregating {} with {}\".format(\n spec['groupby'], spec['select'], agg_name\n ))\n \n # Unique list of features to select\n all_features = list(set(spec['groupby'] + [spec['select']]))\n # Perform the groupby\n gp = X_test[all_features]. \\\n groupby(spec['groupby'])[spec['select']]. \\\n agg(spec['agg']). \\\n reset_index(). \\\n rename(index=str, columns={spec['select']: new_feature})\n \n # Merge back to X_total\n if 'cumcount' == spec['agg']:\n X_test[new_feature] = gp[0].values\n else:\n X_test= X_test.merge(gp, on=spec['groupby'], how='left')\n \n # Clear memory\n del gp\n gc.collect()\n\nX_test.head()","execution_count":14,"outputs":[{"output_type":"stream","text":"Grouping by ['ip', 'app', 'channel'], and aggregating day with var\nGrouping by ['ip', 'app', 'os'], and aggregating hour with var\nGrouping by ['ip', 'day', 'channel'], and aggregating hour with var\nGrouping by ['ip', 'day', 'hour'], and aggregating channel with count\nGrouping by ['ip', 'app'], and aggregating channel with count\nGrouping by ['ip', 'app', 'os'], and aggregating channel with count\nGrouping by ['ip', 'app', 'day', 'hour'], and aggregating channel with count\nGrouping by ['ip', 'app', 'channel'], and aggregating hour with mean\nGrouping by ['app'], and aggregating ip with AvgViewPerDistinct\nGrouping by ['app'], and aggregating channel with count\nGrouping by ['channel'], and aggregating app with count\nGrouping by ['ip'], and aggregating channel with nunique\nGrouping by ['ip'], and aggregating app with nunique\nGrouping by ['ip', 'day'], and aggregating hour with nunique\nGrouping by ['ip', 'app'], and aggregating os with nunique\nGrouping by ['ip'], and aggregating device with nunique\nGrouping by ['app'], and aggregating channel with nunique\nGrouping by ['ip', 'device', 'os'], and aggregating app with nunique\nGrouping by ['ip', 'device', 'os'], and aggregating app with cumcount\nGrouping by ['ip'], and aggregating app with cumcount\nGrouping by ['ip'], and aggregating os with cumcount\nGrouping by ['ip', 'day', 'channel'], and aggregating hour with var\n","name":"stdout"},{"output_type":"execute_result","execution_count":14,"data":{"text/plain":" ip ... ip_day_channel_var_hour_y\n0 112 ... 0.0\n1 93 ... 0.0\n2 95 ... 0.0\n3 -115 ... 0.0\n4 -56 ... 0.0\n\n[5 rows x 32 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timedayhourminutesecondip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_y
01129131072017-11-10 04:00:00104000.00.00.038057657459.992188153586042732611822560000.0
1939134662017-11-10 04:00:00104000.00.00.036572272459.992188153582722712712042580000.0
295211191282017-11-10 04:00:00104000.00.00.046512512413.4140623434305176241742170000.0
3-115151131112017-11-10 04:00:00104000.00.00.041124924424.3359386230361772719322180000.0
4-56121133282017-11-10 04:00:00104000.00.00.0407531453450.7226561298510607123119526170000.0
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Data\nGROUP_BY_NEXT_CLICKS = [\n \n # Group-1\n {'groupby': ['ip']},\n {'groupby': ['ip', 'app']},\n {'groupby': ['ip', 'channel']},\n {'groupby': ['ip', 'os']},\n \n # Group-3\n {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n {'groupby': ['ip', 'os', 'device']},\n {'groupby': ['ip', 'os', 'device', 'app']}\n]\n\n# Calculate the time to next click for each group\nfor t in GROUP_BY_NEXT_CLICKS:\n \n # Name of new feature\n new_feature = '{}_nextClick'.format('_'.join(t['groupby'])) \n \n # Unique list of features to select\n all_features = t['groupby'] + ['click_time']\n \n # Run calculation\n print(f\">> Grouping by {t['groupby']}, and saving time to next click in: {new_feature}\")\n X_train[new_feature] = X_train[all_features].groupby(t['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds\n \nX_train.head()","execution_count":15,"outputs":[{"output_type":"stream","text":">> Grouping by ['ip'], and saving time to next click in: ip_nextClick\n>> Grouping by ['ip', 'app'], and saving time to next click in: ip_app_nextClick\n>> Grouping by ['ip', 'channel'], and saving time to next click in: ip_channel_nextClick\n>> Grouping by ['ip', 'os'], and saving time to next click in: ip_os_nextClick\n>> Grouping by ['ip', 'app', 'device', 'os', 'channel'], and saving time to next click in: ip_app_device_os_channel_nextClick\n>> Grouping by ['ip', 'os', 'device'], and saving time to next click in: ip_os_device_nextClick\n>> Grouping by ['ip', 'os', 'device', 'app'], and saving time to next click in: ip_os_device_app_nextClick\n","name":"stdout"},{"output_type":"execute_result","execution_count":15,"data":{"text/plain":" ip ... ip_os_device_app_nextClick\n0 30 ... 4838.0\n1 -51 ... 5205.0\n2 -30 ... 5182.0\n3 -79 ... 5109.0\n4 -17 ... 5098.0\n\n[5 rows x 51 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timeis_attributeddayhourminutesecondip_confRateapp_confRatedevice_confRateos_confRatechannel_confRateapp_channel_confRateapp_os_confRateapp_device_confRatechannel_os_confRatechannel_device_confRateos_device_confRateip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_yip_nextClickip_app_nextClickip_channel_nextClickip_os_nextClickip_app_device_os_channel_nextClickip_os_device_nextClickip_os_device_app_nextClick
03031133792017-11-06 14:32:21061432210.0013060.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.3589740.70000024013214.842.6406251091613536928313429140000.700000946.0946.0946.04838.04838.04838.04838.0
1-5131193792017-11-06 14:33:34061433340.0000000.0002960.0013070.0013890.00.00.000000.0003060.00.00.0014290.00.2666670.50000015515115.542.6406251091613536823314429160000.5000005177.05177.05177.05196.05263.05196.05205.0
2-3031133792017-11-06 14:34:12061434120.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.5714290.9166671287114.542.6406251091613536025313329150000.9166674574.04574.04574.05150.0NaN5150.05182.0
3-79141134782017-11-06 14:34:52061434520.0006710.0006850.0013070.0011490.00.00.000000.0006980.00.00.001187NaN0.4444441.3333331319114.011.92578130531267431311519210001.3333333948.05109.05137.04086.0NaN4086.05109.0
4-1731133792017-11-06 14:35:0806143580.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.2500000.57142916216115.042.6406251091613536928220429190000.5714295092.05093.05098.05094.05098.05094.05098.0
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Test Data\nGROUP_BY_NEXT_CLICKS = [\n \n # V1\n {'groupby': ['ip']},\n {'groupby': ['ip', 'app']},\n {'groupby': ['ip', 'channel']},\n {'groupby': ['ip', 'os']},\n \n # V3\n {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n {'groupby': ['ip', 'os', 'device']},\n {'groupby': ['ip', 'os', 'device', 'app']}\n]\n\n# Calculate the time to next click for each group\nfor t in GROUP_BY_NEXT_CLICKS:\n \n # Name of new feature\n new_feature = '{}_nextClick'.format('_'.join(t['groupby'])) \n \n # Unique list of features to select\n all_features = t['groupby'] + ['click_time']\n \n # Run calculation\n print(f\">> Grouping by {t['groupby']}, and saving time to next click in: {new_feature}\")\n X_test[new_feature] = X_test[all_features].groupby(t['groupby']).click_time.transform(lambda x: x.diff().shift(-1)).dt.seconds\n \nX_test.head()","execution_count":16,"outputs":[{"output_type":"stream","text":">> Grouping by ['ip'], and saving time to next click in: ip_nextClick\n>> Grouping by ['ip', 'app'], and saving time to next click in: ip_app_nextClick\n>> Grouping by ['ip', 'channel'], and saving time to next click in: ip_channel_nextClick\n>> Grouping by ['ip', 'os'], and saving time to next click in: ip_os_nextClick\n>> Grouping by ['ip', 'app', 'device', 'os', 'channel'], and saving time to next click in: ip_app_device_os_channel_nextClick\n>> Grouping by ['ip', 'os', 'device'], and saving time to next click in: ip_os_device_nextClick\n>> Grouping by ['ip', 'os', 'device', 'app'], and saving time to next click in: ip_os_device_app_nextClick\n","name":"stdout"},{"output_type":"execute_result","execution_count":16,"data":{"text/plain":" ip ... ip_os_device_app_nextClick\n0 112 ... 20.0\n1 93 ... 57.0\n2 95 ... 16.0\n3 -115 ... 11.0\n4 -56 ... 7.0\n\n[5 rows x 39 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timedayhourminutesecondip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_yip_nextClickip_app_nextClickip_channel_nextClickip_os_nextClickip_app_device_os_channel_nextClickip_os_device_nextClickip_os_device_app_nextClick
01129131072017-11-10 04:00:00104000.00.00.038057657459.992188153586042732611822560000.00.04.00.06.0NaN6.020.0
1939134662017-11-10 04:00:00104000.00.00.036572272459.992188153582722712712042580000.00.01.022.02.0NaN2.057.0
295211191282017-11-10 04:00:00104000.00.00.046512512413.4140623434305176241742170000.00.09.015.04.088.04.016.0
3-115151131112017-11-10 04:00:00104000.00.00.041124924424.3359386230361772719322180000.00.06.068.00.0NaN0.011.0
4-56121133282017-11-10 04:00:00104000.00.00.0407531453450.7226561298510607123119526170000.00.02.05.00.0NaN0.07.0
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"HISTORY_CLICKS = {\n 'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],\n 'app_clicks': ['ip', 'app']\n}\n\n# Go through different group-by combinations\nfor fname, fset in HISTORY_CLICKS.items():\n \n # Clicks in the past\n X_train['prev_'+fname] = X_train. \\\n groupby(fset). \\\n cumcount(). \\\n rename('prev_'+fname)\n \n # Clicks in the future\n X_train['future_'+fname] = X_train.iloc[::-1]. \\\n groupby(fset). \\\n cumcount(). \\\n rename('future_'+fname).iloc[::-1]\n\n# Count cumulative subsequent clicks\nX_train.head()\n","execution_count":17,"outputs":[{"output_type":"execute_result","execution_count":17,"data":{"text/plain":" ip app ... prev_app_clicks future_app_clicks\n0 30 3 ... 0 39\n1 -51 3 ... 0 54\n2 -30 3 ... 0 27\n3 -79 14 ... 0 30\n4 -17 3 ... 0 61\n\n[5 rows x 55 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timeis_attributeddayhourminutesecondip_confRateapp_confRatedevice_confRateos_confRatechannel_confRateapp_channel_confRateapp_os_confRateapp_device_confRatechannel_os_confRatechannel_device_confRateos_device_confRateip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_yip_nextClickip_app_nextClickip_channel_nextClickip_os_nextClickip_app_device_os_channel_nextClickip_os_device_nextClickip_os_device_app_nextClickprev_identical_clicksfuture_identical_clicksprev_app_clicksfuture_app_clicks
03031133792017-11-06 14:32:21061432210.0013060.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.3589740.70000024013214.842.6406251091613536928313429140000.700000946.0946.0946.04838.04838.04838.04838.001039
1-5131193792017-11-06 14:33:34061433340.0000000.0002960.0013070.0013890.00.00.000000.0003060.00.00.0014290.00.2666670.50000015515115.542.6406251091613536823314429160000.5000005177.05177.05177.05196.05263.05196.05205.002054
2-3031133792017-11-06 14:34:12061434120.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.5714290.9166671287114.542.6406251091613536025313329150000.9166674574.04574.04574.05150.0NaN5150.05182.000027
3-79141134782017-11-06 14:34:52061434520.0006710.0006850.0013070.0011490.00.00.000000.0006980.00.00.001187NaN0.4444441.3333331319114.011.92578130531267431311519210001.3333333948.05109.05137.04086.0NaN4086.05109.000030
4-1731133792017-11-06 14:35:0806143580.0000000.0002960.0013070.0011490.00.00.000270.0003060.00.00.0011870.00.2500000.57142916216115.042.6406251091613536928220429190000.5714295092.05093.05098.05094.05098.05094.05098.001061
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test Data\nHISTORY_CLICKS = {\n 'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],\n 'app_clicks': ['ip', 'app']\n}\n\n# Go through different group-by combinations\nfor fname, fset in HISTORY_CLICKS.items():\n \n # Clicks in the past\n X_test['prev_'+fname] = X_test. \\\n groupby(fset). \\\n cumcount(). \\\n rename('prev_'+fname)\n \n # Clicks in the future\n X_test['future_'+fname] = X_test.iloc[::-1]. \\\n groupby(fset). \\\n cumcount(). \\\n rename('future_'+fname).iloc[::-1]\n\n# Count cumulative subsequent clicks\nX_test.head()\n","execution_count":18,"outputs":[{"output_type":"execute_result","execution_count":18,"data":{"text/plain":" ip app ... prev_app_clicks future_app_clicks\n0 112 9 ... 0 56\n1 93 9 ... 0 71\n2 95 21 ... 0 11\n3 -115 15 ... 0 23\n4 -56 12 ... 0 52\n\n[5 rows x 43 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
ipappdeviceoschannelclick_timedayhourminutesecondip_app_channel_var_dayip_app_os_var_hourip_day_channel_var_hour_xip_day_hour_count_channelip_app_count_channelip_app_os_count_channelip_app_day_hour_count_channelip_app_channel_mean_hourapp_AvgViewPerDistinct_ipapp_count_channelchannel_count_appip_nunique_channelip_nunique_appip_day_nunique_hourip_app_nunique_osip_nunique_deviceapp_nunique_channelip_device_os_nunique_appip_device_os_cumcount_appip_cumcount_appip_cumcount_osip_day_channel_var_hour_yip_nextClickip_app_nextClickip_channel_nextClickip_os_nextClickip_app_device_os_channel_nextClickip_os_device_nextClickip_os_device_app_nextClickprev_identical_clicksfuture_identical_clicksprev_app_clicksfuture_app_clicks
01129131072017-11-10 04:00:00104000.00.00.038057657459.992188153586042732611822560000.00.04.00.06.0NaN6.020.000056
1939134662017-11-10 04:00:00104000.00.00.036572272459.992188153582722712712042580000.00.01.022.02.0NaN2.057.000071
295211191282017-11-10 04:00:00104000.00.00.046512512413.4140623434305176241742170000.00.09.015.04.088.04.016.001011
3-115151131112017-11-10 04:00:00104000.00.00.041124924424.3359386230361772719322180000.00.06.068.00.0NaN0.011.000023
4-56121133282017-11-10 04:00:00104000.00.00.0407531453450.7226561298510607123119526170000.00.02.05.00.0NaN0.07.000052
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split into X and y\nX_train.fillna(X_train.mean(), inplace=True)\ny_train = X_train['is_attributed']\nX_train= X_train.drop('is_attributed', axis=1).select_dtypes(include=[np.number])\n# Oversampling to decrease imbalance in labels\nfrom imblearn.over_sampling import SMOTE\nsm = SMOTE()\nX_train, y_train = sm.fit_sample(X_train, y_train)\n#Shuffle the data to train well\nfrom sklearn.utils import shuffle\nshuffle(X_train)\n","execution_count":19,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"},{"output_type":"execute_result","execution_count":19,"data":{"text/plain":"array([[-77.63441447, 26.46497408, 0.74649741, ..., 0. ,\n 0.74649741, 0. ],\n [-73.62323071, 26.57941079, 0.75794108, ..., 0.24205892,\n 1. , 0.24205892],\n [-50. , 12. , 1. , ..., 0. ,\n 54. , 48. ],\n ...,\n [-95.80193453, 8. , 1. , ..., 0.16038691,\n 17.3632887 , 3.48116072],\n [127. , 12. , 1. , ..., 0. ,\n 22. , 13. ],\n [-13.47870995, 19. , 0. , ..., 0. ,\n 2.82264855, 0. ]])"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import chi2\nX_train = SelectKBest(chi2, k=42).fit_transform(abs(X_train), y_train)\nX_train.shape","execution_count":20,"outputs":[{"output_type":"execute_result","execution_count":20,"data":{"text/plain":"(199662, 42)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid= train_test_split(X_train,y_train,test_size=0.3, random_state=0)","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a model\n# Params from: https://www.kaggle.com/aharless/swetha-s-xgboost-revised\nimport xgboost as xgb\nclf_xgBoost = xgb.XGBClassifier(max_depth = 4,subsample = 0.8,colsample_bytree = 0.7,colsample_bylevel = 0.7,scale_pos_weight = 9,\n min_child_weight = 0,reg_alpha = 0.01,n_jobs = -1, objective = 'binary:logistic')\n# Fit the models\nclf_xgBoost.fit(X_train, y_train)","execution_count":22,"outputs":[{"output_type":"execute_result","execution_count":22,"data":{"text/plain":"XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n colsample_bytree=0.7, gamma=0, learning_rate=0.1, max_delta_step=0,\n max_depth=4, min_child_weight=0, missing=None, n_estimators=100,\n n_jobs=-1, nthread=None, objective='binary:logistic',\n random_state=0, reg_alpha=0.01, reg_lambda=1, scale_pos_weight=9,\n seed=None, silent=True, subsample=0.8)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ny_pred=clf_xgBoost.predict(X_valid)","execution_count":23,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(y_valid, y_pred)","execution_count":24,"outputs":[{"output_type":"execute_result","execution_count":24,"data":{"text/plain":"0.9946728840355252"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nclf1=RandomForestClassifier(n_jobs=-1,criterion=\"entropy\",min_samples_leaf=1,min_samples_split=8, \\\n n_estimators=15,max_features=None,random_state=100)\nclf1.fit(X_train,y_train)\n \n \n \ny_pred_rf=clf1.predict(X_valid)\nprint(y_pred_rf)\n \n\n\n","execution_count":25,"outputs":[{"output_type":"stream","text":"[1 1 1 ... 1 1 0]\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(y_valid, y_pred_rf)","execution_count":26,"outputs":[{"output_type":"execute_result","execution_count":26,"data":{"text/plain":"0.9990634346346802"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.drop('click_time',axis=1,inplace=True)","execution_count":27,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.fillna(X_test.mean(), inplace=True)\n","execution_count":28,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_t=clf1.predict(X_test)","execution_count":29,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create submission file\nsubmission = pd.DataFrame({'click_id':[i for i in range(len(y_pred_t))],'is_attributed':y_pred_t})\nsubmission.to_csv('submission.csv', index=False)","execution_count":30,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}