{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport datetime\nimport os\n\nimport time\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(12,5)});\nplt.figure(figsize=(12,5));","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', nrows=10000000)\ntest = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"variables = ['ip', 'app', 'device', 'os', 'channel']\nfor v in variables:\n    train[v] = train[v].astype('category')\n    test[v]=test[v].astype('category')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#set click_time and attributed_time as timeseries\ntrain['click_time'] = pd.to_datetime(train['click_time'])\ntrain['attributed_time'] = pd.to_datetime(train['attributed_time'])\ntest['click_time'] = pd.to_datetime(test['click_time'])\n\n#set as_attributed in train as a categorical\ntrain['is_attributed']=train['is_attributed'].astype('category')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(train[col].unique()) for col in cols]\nsns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, log=True)\nax.set(xlabel='Feature', ylabel='log(unique count)', title='Number of unique values per feature (from 10,000,000 samples)')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\") \n# for col, uniq in zip(cols, uniques):\n#     ax.text(col, uniq, uniq, color='black', ha=\"center\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#double check that 'attributed_time' is not Null for all values that resulted in download (i.e. is_attributed == 1)\ntrain[['attributed_time', 'is_attributed']][train['is_attributed']==1].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#set click_id to categorical, for cleaner statistics view\ntest['click_id']=test['click_id'].astype('category')\ntest.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Quick Notes/Observations :\n\n* There are only 18717 attributed_time values. This means only 18,717 out of 10,000,000 clicks resulted in a download. That's less than 0.2% !\n* There are ip adresses that trigger a click over 50 thousand times. Seems strange that one ip address would click that often in a span of just 4 days. Does that mean that ip address encoded is not device id, but network id? (explore this below)\n* First click in train set is on 2017-11-06 14:32:21. Test clicks start on 2017-11-10. Based on data specifications, train coveres a 4 day period. This means that the train and test data do not overlap, but test data is taken the day after train data ends. -Train data is ordered by timestamp. (therefore batches pulled in order cover limited time span)\n* 2017-11-06 was a Monday. 2017-11-10 was a Friday. i.e. Train is Mon-Thur, Test is Friday -There is no missing data in Test. Missing values in train appear to be only for attributed_time, where there isn't any value due to no app download.","metadata":{}},{"cell_type":"markdown","source":"Only a small proportion of clicks were followed by a download:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6,6))\n#sns.set(font_scale=1.2)\nmean = (train.is_attributed.values == 1).mean()\nax = sns.barplot(['App Downloaded (1)', 'Not Downloaded (0)'], [mean, 1-mean])\nax.set(ylabel='Proportion', title='App Downloaded vs Not Downloaded')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Explore ip counts. Check if multiple ips have any downloads.**\n\nAt this point I was trying to figure out what 'ip' were actually encoding. My original understanding that ips were user specific did not hold up to scrutiny. If ip repeated too many times, was it a bot? This does not appear to be true, as repeated ips do convert. See below:","metadata":{}},{"cell_type":"code","source":"#temporary table to see ips with their associated count frequencies\ntemp = train['ip'].value_counts().reset_index(name='counts')\ntemp.columns = ['ip', 'counts']\ntemp[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add temporary counts of ip feature ('counts') to the train table, to see if IPs with high counts have conversions\ntrain= train.merge(temp, on='ip', how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check top 10 values\ntrain[train['is_attributed']==1].sort_values('counts', ascending=False)[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['is_attributed']==1].ip.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So high frequency ip counts do get conversions. Up to 56 downloads for one ip. Each IP must be for some network with many devices.","metadata":{}},{"cell_type":"code","source":"#convert 'is_attributed' back to numeric for proportion calculations\ntrain['is_attributed']=train['is_attributed'].astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conversion rates over Counts of 300 most popular IPs**","metadata":{}},{"cell_type":"code","source":"proportion = train[['ip', 'is_attributed']].groupby('ip', as_index=False).mean().sort_values('is_attributed', ascending=False)\ncounts = train[['ip', 'is_attributed']].groupby('ip', as_index=False).count().sort_values('is_attributed', ascending=False)\nmerge = counts.merge(proportion, on='ip', how='left')\nmerge.columns = ['ip', 'click_count', 'prop_downloaded']\n\nax = merge[:300].plot(secondary_y='prop_downloaded')\nplt.title('Conversion Rates over Counts of 300 Most Popular IPs')\nax.set(ylabel='Count of clicks')\nplt.ylabel('Proportion Downloaded')\nplt.show()\n\nprint('Counversion Rates over Counts of Most Popular IPs')\nprint(merge[:20])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Conversions are noisy and do not appear to correlate with how popular an IP is.","metadata":{}},{"cell_type":"markdown","source":"**Conversions by App**\n\nCheck 100 most popular apps by click count:","metadata":{}},{"cell_type":"code","source":"proportion = train[['app', 'is_attributed']].groupby('app', as_index=False).mean().sort_values('is_attributed', ascending=False)\ncounts = train[['app', 'is_attributed']].groupby('app', as_index=False).count().sort_values('is_attributed', ascending=False)\nmerge = counts.merge(proportion, on='app', how='left')\nmerge.columns = ['app', 'click_count', 'prop_downloaded']\n\nax = merge[:100].plot(secondary_y='prop_downloaded')\nplt.title('Conversion Rates over Counts of 100 Most Popular Apps')\nax.set(ylabel='Count of clicks')\nplt.ylabel('Proportion Downloaded')\nplt.show()\n\nprint('Counversion Rates over Counts of Most Popular Apps')\nprint(merge[:20])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a again a huge difference in clicks per app, with minimum of one click on an app and max at almost 13 million. The proportion flucuates more as the counts go down, since each additional click has larger impact on the proportion value. In general, for apps with counts in the thousands the ratio stays within 0.0001 - 0.0015 boundary. For less popular apps it fluxuates more widely.","metadata":{}},{"cell_type":"markdown","source":"**Conversions by OS**\n\nLook at top 100 operating systems by click count","metadata":{}},{"cell_type":"code","source":"proportion = train[['os', 'is_attributed']].groupby('os', as_index=False).mean().sort_values('is_attributed', ascending=False)\ncounts = train[['os', 'is_attributed']].groupby('os', as_index=False).count().sort_values('is_attributed', ascending=False)\nmerge = counts.merge(proportion, on='os', how='left')\nmerge.columns = ['os', 'click_count', 'prop_downloaded']\n\nax = merge[:100].plot(secondary_y='prop_downloaded')\nplt.title('Conversion Rates over Counts of 100 Most Popular Operating Systems')\nax.set(ylabel='Count of clicks')\nplt.ylabel('Proportion Downloaded')\nplt.show()\n\nprint('Counversion Rates over Counts of Most Popular Operating Systems')\nprint(merge[:20])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Same story. For values in the thousands the boundary on the ratio is very low, roughly between 0.0006 and 0.003, but as counts on OS become lower, the ratio starts fluxuating more wildely.","metadata":{}},{"cell_type":"markdown","source":"**Conversions by Device**\n\nDevices are extremely disproportionately distributed, with number one device used almost 94% of time. For that device proportion download was 0.001326. (0.13%)","metadata":{}},{"cell_type":"code","source":"proportion = train[['device', 'is_attributed']].groupby('device', as_index=False).mean().sort_values('is_attributed', ascending=False)\ncounts = train[['device', 'is_attributed']].groupby('device', as_index=False).count().sort_values('is_attributed', ascending=False)\nmerge = counts.merge(proportion, on='device', how='left')\nmerge.columns = ['device', 'click_count', 'prop_downloaded']\n\nprint('Count of clicks and proportion of downloads by device:')\nprint(merge)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conversions by Channel\n**","metadata":{}},{"cell_type":"code","source":"proportion = train[['channel', 'is_attributed']].groupby('channel', as_index=False).mean().sort_values('is_attributed', ascending=False)\ncounts = train[['channel', 'is_attributed']].groupby('channel', as_index=False).count().sort_values('is_attributed', ascending=False)\nmerge = counts.merge(proportion, on='channel', how='left')\nmerge.columns = ['channel', 'click_count', 'prop_downloaded']\n\nax = merge[:100].plot(secondary_y='prop_downloaded')\nplt.title('Conversion Rates over Counts of 100 Most Popular Apps')\nax.set(ylabel='Count of clicks')\nplt.ylabel('Proportion Downloaded')\nplt.show()\n\nprint('Counversion Rates over Counts of Most Popular Channels')\nprint(merge[:20])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There appear to be a few peaks for channels at reasonable click quantity, but overall the pattern holds same as for categories above.","metadata":{}},{"cell_type":"markdown","source":"# ****Checking for time patterns****\n\nRound the click time down to an hour of the day to see if there are any hourly patterns.\n\nFor this part cannot use the first n rows from train data, as it's organized by time. To get a genral idea for the pattern, will use train data from the randomly sampled 100000 train set provided by organizers.","metadata":{}},{"cell_type":"code","source":"train_smp = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train_sample.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smp.head(7)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert click_time and attributed_time to time series\ntrain_smp['click_time'] = pd.to_datetime(train_smp['click_time'])\ntrain_smp['attributed_time'] = pd.to_datetime(train_smp['attributed_time'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#round the time to nearest hour\ntrain_smp['click_rnd']=train_smp['click_time'].dt.round('H')  \n\n#check for hourly patterns\ntrain_smp[['click_rnd','is_attributed']].groupby(['click_rnd'], as_index=True).count().plot()\nplt.title('HOURLY CLICK FREQUENCY');\nplt.ylabel('Number of Clicks');\n\ntrain_smp[['click_rnd','is_attributed']].groupby(['click_rnd'], as_index=True).mean().plot()\nplt.title('HOURLY CONVERSION RATIO');\nplt.ylabel('Converted Ratio');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no clear hourly time pattern in ratios, however there is a definete pattern in frequency of clicks based on time of day.\n\nLets extract the hour of day from each day as a separate feature, and see combined trend (merge the 4 days together by hour).","metadata":{}},{"cell_type":"code","source":"#extract hour as a feature\ntrain_smp['click_hour']=train_smp['click_time'].dt.hour","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smp.head(7)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check number of clicks by hour:","metadata":{}},{"cell_type":"code","source":"train_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).count().plot(kind='bar', color='#a675a1')\nplt.title('HOURLY CLICK FREQUENCY Barplot');\nplt.ylabel('Number of Clicks');\n\ntrain_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).count().plot(color='#a675a1')\nplt.title('HOURLY CLICK FREQUENCY Lineplot');\nplt.ylabel('Number of Clicks');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And number of conversions by hours:","metadata":{}},{"cell_type":"code","source":"train_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).mean().plot(kind='bar', color='#75a1a6')\nplt.title('HOURLY CONVERSION RATIO Barplot');\nplt.ylabel('Converted Ratio');\n\ntrain_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).mean().plot( color='#75a1a6')\nplt.title('HOURLY CONVERSION RATIO Lineplot');\nplt.ylabel('Converted Ratio');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"overlay the two graphs to see if patterns correlate in any way","metadata":{}},{"cell_type":"code","source":"group = train_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=False).mean()\nx = group['click_hour']\nymean = group['is_attributed']\ngroup = train_smp[['click_hour','is_attributed']].groupby(['click_hour'], as_index=False).count()\nycount = group['is_attributed']\n\n\nfig = plt.figure()\nhost = fig.add_subplot(111)\npar1 = host.twinx()\n\nhost.set_xlabel(\"Time\")\nhost.set_ylabel(\"Proportion Converted\")\npar1.set_ylabel(\"Click Count\")\n\n#color1 = plt.cm.viridis(0)\n#color2 = plt.cm.viridis(0.5)\ncolor1 = '#75a1a6'\ncolor2 = '#a675a1'\n\np1, = host.plot(x, ymean, color=color1,label=\"Proportion Converted\")\np2, = par1.plot(x, ycount, color=color2, label=\"Click Count\")\n\nlns = [p1, p2]\nhost.legend(handles=lns, loc='best')\n\nhost.yaxis.label.set_color(p1.get_color())\npar1.yaxis.label.set_color(p2.get_color())\n\nplt.savefig(\"pyplot_multiple_y-axis.png\", bbox_inches='tight')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot('click_hour', 'is_attributed', data=train_smp)\nplt.title('HOURLY CONVERSION RATIO');\nplt.ylabel('Converted Ratio');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Look into attributed_time**\n\nIt could be useful to learn more about conversions that did take place. Let's see how much time passed from clicking on the ad to downloading it.","metadata":{}},{"cell_type":"code","source":"train_smp['timePass']= train_smp['attributed_time']-train_smp['click_time']\n#check:\ntrain_smp[train_smp['is_attributed']==1][:15]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smp['timePass'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It takes as long as (almost) 20 hours to go from click to purchase and as little as 4 seconds.\n\nThe 4 seconds seems to low to make a decision. This person would have either seen the ad before, or already been aware of the product some other way.\n\nDoes that mean the ad was clicked on multiple times, but only one click was counted as conversion? Or did the person click on the ad specifically with the intent to download? (eg, if channel is something like google search, the ad could be clicked during search results view and app downloaded immediately because that's what the person intended to do right away)\n\nRaises questions to explore:\n\nHow accurately are conversions tracked? How are clicks and downloads linked? What happens if download after multiple clicks? Is there a way to identify likely same users (same IP, Device, etc...)","metadata":{}},{"cell_type":"code","source":"import os\nimport datetime\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nfrom xgboost import plot_importance","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:35:20.369684Z","iopub.execute_input":"2021-08-10T17:35:20.370093Z","iopub.status.idle":"2021-08-10T17:35:22.416666Z","shell.execute_reply.started":"2021-08-10T17:35:20.370055Z","shell.execute_reply":"2021-08-10T17:35:22.41556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train_sample.csv', parse_dates=['click_time', 'attributed_time'])\ntest = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/test.csv', parse_dates=['click_time'])\n\npath = '../input/talkingdata-adtracking-fraud-detection/'\n","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:40:57.450755Z","iopub.execute_input":"2021-08-10T17:40:57.451225Z","iopub.status.idle":"2021-08-10T17:41:14.91216Z","shell.execute_reply.started":"2021-08-10T17:40:57.451175Z","shell.execute_reply":"2021-08-10T17:41:14.911065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:41:18.711112Z","iopub.execute_input":"2021-08-10T17:41:18.711624Z","iopub.status.idle":"2021-08-10T17:41:18.729089Z","shell.execute_reply.started":"2021-08-10T17:41:18.711589Z","shell.execute_reply":"2021-08-10T17:41:18.72783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:42:49.115276Z","iopub.execute_input":"2021-08-10T17:42:49.116042Z","iopub.status.idle":"2021-08-10T17:43:00.880605Z","shell.execute_reply.started":"2021-08-10T17:42:49.116004Z","shell.execute_reply":"2021-08-10T17:43:00.879458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:42:22.154267Z","iopub.execute_input":"2021-08-10T17:42:22.154645Z","iopub.status.idle":"2021-08-10T17:42:22.164162Z","shell.execute_reply.started":"2021-08-10T17:42:22.154615Z","shell.execute_reply":"2021-08-10T17:42:22.163298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categories = list(train.select_dtypes('int64').columns.values)\ncategories.remove('is_attributed')\n\nfor col in categories:\n    train[col] = train[col].astype('category')\n    test[col] = test[col].astype('category')\n\ntrain.is_attributed = train.is_attributed.astype('category')\ntest.click_id = test.click_id.astype('category')","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:41:28.624494Z","iopub.execute_input":"2021-08-10T17:41:28.624851Z","iopub.status.idle":"2021-08-10T17:41:37.14063Z","shell.execute_reply.started":"2021-08-10T17:41:28.624822Z","shell.execute_reply":"2021-08-10T17:41:37.139497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nmean = (train.is_attributed.values == 1).mean()\nax = sns.barplot(['is_attributed (1)', 'is_attributed (0)'], [mean, 1-mean], palette='deep')\nax.set(xlabel='Target', ylabel='Probability', title='Distribution of is_attribute')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")\n","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:41:37.142732Z","iopub.execute_input":"2021-08-10T17:41:37.143206Z","iopub.status.idle":"2021-08-10T17:41:37.314321Z","shell.execute_reply.started":"2021-08-10T17:41:37.143113Z","shell.execute_reply":"2021-08-10T17:41:37.313211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nuniques = [len(train[col].unique()) for col in categories]\nsns.set(font_scale=1.2)\nax = sns.barplot(categories, uniques, log=True)\nax.set(xlabel='Feature', ylabel='Count', title='Number of unique values per feature')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:42:34.664819Z","iopub.execute_input":"2021-08-10T17:42:34.665395Z","iopub.status.idle":"2021-08-10T17:42:35.062389Z","shell.execute_reply.started":"2021-08-10T17:42:34.665361Z","shell.execute_reply":"2021-08-10T17:42:35.061628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check if all attributed_time are recorded according to is_attributed.\n**","metadata":{}},{"cell_type":"code","source":"print(train.loc[train.is_attributed==1]['attributed_time'].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:42:35.834082Z","iopub.execute_input":"2021-08-10T17:42:35.83487Z","iopub.status.idle":"2021-08-10T17:42:35.844434Z","shell.execute_reply.started":"2021-08-10T17:42:35.834833Z","shell.execute_reply":"2021-08-10T17:42:35.84364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check ip and device count relationship.**","metadata":{}},{"cell_type":"code","source":"train.groupby('ip')['device'].count().describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:42:41.794958Z","iopub.execute_input":"2021-08-10T17:42:41.79573Z","iopub.status.idle":"2021-08-10T17:42:41.813738Z","shell.execute_reply.started":"2021-08-10T17:42:41.795691Z","shell.execute_reply":"2021-08-10T17:42:41.812911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n**Extract clicking time into day, hour, minute:**","metadata":{}},{"cell_type":"code","source":"def processTimeFeatures(df):\n    df['click_day'] = df.click_time.dt.day\n\n    df['click_hour'] = df.click_time.dt.hour\n\n    df['click_minute'] = df.click_time.dt.minute","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:43:10.174838Z","iopub.execute_input":"2021-08-10T17:43:10.175245Z","iopub.status.idle":"2021-08-10T17:43:10.181447Z","shell.execute_reply.started":"2021-08-10T17:43:10.175208Z","shell.execute_reply":"2021-08-10T17:43:10.180367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processTimeFeatures(train)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:43:12.594045Z","iopub.execute_input":"2021-08-10T17:43:12.59445Z","iopub.status.idle":"2021-08-10T17:43:12.656391Z","shell.execute_reply.started":"2021-08-10T17:43:12.594417Z","shell.execute_reply":"2021-08-10T17:43:12.655291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processTimeFeatures(test)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:45:36.115558Z","iopub.execute_input":"2021-08-10T17:45:36.116007Z","iopub.status.idle":"2021-08-10T17:45:36.141506Z","shell.execute_reply.started":"2021-08-10T17:45:36.115967Z","shell.execute_reply":"2021-08-10T17:45:36.140373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Return rate:\n**Is this a one-off or an actually interesting app to this user that he/she continues to look at it?**","metadata":{}},{"cell_type":"code","source":"def getAvgAppClickPerIp(df):\n    df['avg_app_click_by_ip'] = df[['app', 'ip']].groupby('app')['ip'].agg(lambda x: float(len(x)) / len(x.unique()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getAvgAppClickPerIp(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.avg_app_click_by_ip.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time until next click:\n**Is the user in a download spree?**","metadata":{}},{"cell_type":"code","source":"def getTimeToNextClick(df):\n    df['time_to_next_click'] = df[['ip', 'os', 'device', 'channel', 'click_time']]\\\n                                .groupby(['ip', 'os', 'device', 'channel'])['click_time']\\\n                                .transform(lambda x: x.diff().shift(-1).dt.seconds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"getTimeToNextClick(train)\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.time_to_next_click.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Preprocessing data and write to files for future reuse**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', skiprows=range(1, 84903892), nrows=60000000, parse_dates=['click_time', 'attributed_time'])\n\ngetAvgAppClickPerIp(train)\ngetTimeToNextClick(train)\n\ntrain.to_csv('train_60mil_with_avgClick_timeToNext.csv')\n\ndel train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/test.csv', parse_dates=['click_time'])\n\ngetAvgAppClickPerIp(train)\ngetTimeToNextClick(train)\n\ntest.to_csv('test_with_avgClick_timeToNext.csv')\n\ndel test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and test model\n**Use xgboost.train instead of xgboost.XGBClassifier to enable incremental training.**","metadata":{}},{"cell_type":"code","source":"import time\nimport datetime\nimport xgboost as xgb\n\nnum_rows = 60000000\nstart = 0\nbatch_size = 20000000\n\nparams = {\n    'max_depth': 6,\n    'learning_rate': 0.15,\n    'objective': 'binary:logistic',\n    'n_jobs': 3,\n    'random_state': 42,\n    'gamma': 0.03,\n    'min_child_weight': 4,\n    'max_delta_step': 20,\n    'colsample_bytree': 0.7,\n    'reg_lambda': 1000,\n    'scale_pos_weight': 99, # Because 99 percent of data is negative\n    'tree_method': 'gpu_hist',\n    'predictor':'cpu_predictor' # To avoid Windows Error\n}\n\nfeats = [ # features to train\n    'ip',\n    'app',\n    'device',\n    'os',\n    'channel',\n    'click_day',\n    'click_hour',\n    'click_minute',\n    'avg_app_click_by_ip'\n]\n\nmodel = None\n# model_filename = None\ncur_done = start\n# for cur_done in range(start, num_rows, batch_size):\nwhile cur_done < num_rows:\n    \n    start_time = time.time()\n    \n    train = pd.read_csv('train_60mil_with_avgClick_timeToNext.csv', skiprows=range(start+1, cur_done+1), nrows=batch_size, parse_dates=['click_time', 'attributed_time'])\n\n    train = processTimeFeatures(train)\n    \n    dmatrix = xgb.DMatrix(train[feats], train['is_attributed'], feature_names=feats)\n    model = xgb.train(params, dmatrix, xgb_model=model, num_boost_round=100) # num_boost_round equivalent to n_estimators in XGBClassifier\n#     model_filename = 'models/model'\n#     model.save_model(model_filename)\n    \n    del train\n    \n    elapsed = time.time() - start_time\n    \n    cur_done += batch_size\n    left = num_rows - cur_done\n    if left == 0:\n        break\n    batch_size = min(left, batch_size)\n\n    print(\"\\rProgress = {:9d}, batch size = {:8d}, left = {:9d}, elapsed = {:s}\".format(\n        cur_done, batch_size, left,str(datetime.timedelta(seconds=elapsed))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('test_with_avgClick_timeToNext.csv', parse_dates=['click_time'])\nprocessTimeFeatures(test)\n\npred = model.predict(xgb.DMatrix(test[feats], feature_names=feats))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame(pred, np.array(test['click_id']), columns=['is_attributed'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('xgb_with_xgb_params_acpi_ttnc_60mil.csv', index_label=['click_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\npickle.dump(model, open('models/xgb_with_xgb_params_acpi_ttnc_60mil_1806190601.pickle.dat', 'wb'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature importance","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', nrows=10000000, parse_dates=['click_time', 'attributed_time'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport time\nimport datetime\n\ngroups = [    \n    {'groupby': ['ip']},\n    {'groupby': ['ip', 'app']},\n    {'groupby': ['ip', 'channel']},\n    {'groupby': ['ip', 'os']},\n    {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    {'groupby': ['ip', 'os', 'device']},\n    {'groupby': ['ip', 'os', 'device', 'app']}\n]\n\n# Calculate the time to next click for each group\nfor gr in groups:\n    start_time = time.time()\n    \n    feature_name = '{}_time_to_next_click'.format('_'.join(gr['groupby']))        \n    all_features = gr['groupby'] + ['click_time']\n\n    train[feature_name] = train[gr['groupby'] + ['click_time']].\\\n                            groupby(gr['groupby']).click_time.\\\n                            transform(lambda x: x.diff().shift(-1)).dt.seconds\n    elapsed = time.time() - start_time\n    \n    print('Done {:s}, in {:s}'.format(feature_name, str(datetime.timedelta(seconds=elapsed))))\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('../input/talkingdata-adtracking-fraud-detection/train_time_features_10mil.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    'ip',\n    'app',\n    'device',\n    'os',\n    'channel',\n    'ip_time_to_next_click',\n    'ip_app_time_to_next_click',\n    'ip_channel_time_to_next_click',\n    'ip_os_time_to_next_click',\n    'ip_app_device_os_channel_time_to_next_click',\n    'ip_os_device_time_to_next_click',\n    'ip_os_device_app_time_to_next_click'\n]\n\nmodel = xgb.XGBClassifier()\nmodel.fit(train[features], train['is_attributed'])\n\nxgb.plot_importance(model)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}