{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport dask.dataframe as dd\nimport gc\n\n# Lots of stuff in here copied from other kernels. Not sure who to credit because I saw them in \n# multiple places but I'll try to mark them\n\n\n# Specify data types for the columns to save space\n# Taken from a thread on advice for saving memory\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\n\n# Read input, use dask to sample a fraction rather than doing first n rows\ndata = dd.read_csv('../input/train.csv', dtype=dtypes, \n                  usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed']).sample(0.1).compute()\n\ntrain_len = len(data)\nprint(train_len)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","scrolled":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"collapsed":true},"cell_type":"code","source":"# Load test data, append to train\n# Use this larger collection for counting features by IP and other columns\n# Idea taken from some other kernel\ntest_data = pd.read_csv(\"../input/test.csv\", dtype=dtypes)\ndata = data.append(test_data)\ndel test_data\ngc.collect()\n\nprint(\"Total rows: {:d}\".format(len(data)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fdde292c-ad06-46a6-ba44-294ba67fc3c4","_uuid":"2df04ebd28bfcf29430d623aeefd8838ded41f66"},"cell_type":"markdown","source":"Let's pull out some time data and plot it"},{"metadata":{"_cell_guid":"7c504b66-32a2-413c-88b9-a48616a253cc","collapsed":true,"_uuid":"8ee845e3eeefa6906143da14db4fa191e1541d57","trusted":false},"cell_type":"code","source":"    data['click_time'] = dd.to_datetime(data['click_time'])\n    data['hour'] = data['click_time'].dt.hour\n    data['minute'] = data['click_time'].dt.minute\n    data['second'] = data['click_time'].dt.second\n    data['day'] = data['click_time'].dt.day\n    \n    data = data.drop(columns=['click_time'])\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1c460f49-1586-4263-8e3f-d2fb2555e1f9","_uuid":"b976323da88705db0e24ae65312ff03b78864e02","trusted":false,"collapsed":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndata.hist('hour', by='is_attributed', bins=24, normed=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c0df1b0c-6d3f-4113-8268-f7bf30b26beb","_uuid":"0aaf7ec956d9e34973aace94bbff5d3ce89a96d3","trusted":false,"collapsed":true},"cell_type":"code","source":"data.hist('minute', by='is_attributed', bins=60, normed=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f3dc9a32-7f64-4288-921c-0e97b6fd5938","_uuid":"fccd6d82f5daaffb498f7cf59ced5d00cd5edd1e"},"cell_type":"markdown","source":"Look at that! Another big peak in the unattributed data. Looks like there are a bunch of bots going off on a chron script or something, right on the first minute of every hour."},{"metadata":{"_cell_guid":"87b27c52-5e26-446b-af4e-0de8f7cfdc37","_uuid":"34cffc55f86f94bc8dd600215b39416af16a9989","trusted":false,"collapsed":true},"cell_type":"code","source":"data.hist('second', by='is_attributed', bins=60, normed=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"889248f1-edbd-4880-95ab-754951c99816","_uuid":"06171e1b8a5dcbee1a3780a37bca270623d71358"},"cell_type":"markdown","source":"Not as clear of a spike in the seconds.\n\nLet's try using that spike in the minutes as a feature. Just using a flag for each value didn't seem to work, so how about a count by IP\n\n"},{"metadata":{"_cell_guid":"b69d933b-ac28-4c8e-bf23-3eeadb591e3d","_uuid":"a820cbef6bf42e7ad8f90bb13d4882a2c6bd27a7","trusted":false,"collapsed":true},"cell_type":"code","source":"data['minute_0'] = (data['minute']==0)\n\ngp = data[['ip', 'minute_0']].groupby(by=['ip']).count().reset_index().rename(columns={'minute_0':'minute_0_count'})\ndata = data.merge(gp, on='ip', how='left')\ndel gp\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a8752dc0-04c5-46e4-9986-aaaf6dbc9548","_uuid":"cb6e32dced4f86a9faf23043b477176b11bbcbb2","trusted":false,"collapsed":true},"cell_type":"code","source":"data[data.minute_0_count > 10000].hist('minute_0_count', by='is_attributed', bins=10, normed=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73e799d528e144101e183b750f74179670bd665d"},"cell_type":"markdown","source":"Not seeing anything useful here. How about a more general check on potential repeating schedules"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e873ce406a8d06d7a4e22e22705787a3b4f22a53"},"cell_type":"code","source":"# Smallest possible frequency this could have been scheduled for if on a repeating schedule: 60/gcd(60,x)\nminInterval = {0: 1.0, 1: 60.0, 2: 30.0, 3: 20.0, 4: 15.0, 5: 12.0, 6: 10.0, 7: 60.0, 8: 15.0, 9: 20.0, 10: 6.0, 11: 60.0,\n12: 5.0, 13: 60.0, 14: 30.0, 15: 4.0, 16: 15.0, 17: 60.0, 18: 10.0, 19: 60.0, 20: 3.0, 21: 20.0, 22: 30.0, 23: 60.0, 24: 5.0, \n25: 12.0, 26: 30.0, 27: 20.0, 28: 15.0, 29: 60.0, 30: 2.0, 31: 60.0, 32: 15.0, 33: 20.0, 34: 30.0, 35: 12.0, 36: 5.0, 37: 60.0, \n38: 30.0, 39: 20.0, 40: 3.0, 41: 60.0, 42: 10.0, 43: 60.0, 44: 15.0, 45: 4.0, 46: 30.0, 47: 60.0, 48: 5.0, 49: 60.0, 50: 6.0, \n51: 20.0, 52: 15.0, 53: 60.0, 54: 10.0, 55: 12.0, 56: 15.0, 57: 20.0, 58: 30.0, 59: 60.0}\ndata['minute_interval'] = data['minute'].map(minInterval)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9f98a05a913de856b95ee99c3acae581b6b25b0d"},"cell_type":"code","source":"gp = data[['ip','os','app','minute_interval']].groupby(by=['ip','os','app']).mean().reset_index().rename(columns={'minute_interval':'minute_interval_avg'})\ndata = data.merge(gp, on=['ip','os','app'], how='left')\ndel gp\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cef6bec3d5b6db0dfbf4fa9cba1d96c10bd864a5"},"cell_type":"code","source":"data.hist('minute_interval', by='is_attributed')\ndata.hist('minute_interval_avg', by='is_attributed')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c2f520df-be67-45d0-b7f1-fc2eec554fd8","_uuid":"ddb0e6fd5ce1e3623d1f76837567f6855e506146"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}