{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-10T21:02:33.402244Z","iopub.execute_input":"2021-08-10T21:02:33.402706Z","iopub.status.idle":"2021-08-10T21:02:33.412296Z","shell.execute_reply.started":"2021-08-10T21:02:33.402611Z","shell.execute_reply":"2021-08-10T21:02:33.411508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport gc\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\nclick_data = pd.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv',parse_dates=['click_time', 'attributed_time'], dtype=dtypes, skiprows=range(1,122991234), nrows=60000000).sample(n=1000000)\nprint(\"Reading Training data... Done\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:02:33.413367Z","iopub.execute_input":"2021-08-10T21:02:33.413737Z","iopub.status.idle":"2021-08-10T21:05:29.831327Z","shell.execute_reply.started":"2021-08-10T21:02:33.413709Z","shell.execute_reply":"2021-08-10T21:05:29.830229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = ['ip', 'app', 'device', 'os', 'channel', 'is_attributed']\nfor c in columns:\n    click_data[c] = click_data[c].astype('category')\nclick_data.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:29.833101Z","iopub.execute_input":"2021-08-10T21:05:29.833406Z","iopub.status.idle":"2021-08-10T21:05:30.162991Z","shell.execute_reply.started":"2021-08-10T21:05:29.833376Z","shell.execute_reply":"2021-08-10T21:05:30.161552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nplt.figure(figsize=(10, 6))\ntrain_columns = ['ip', 'app', 'device', 'os', 'channel']\nunique_count = [len(click_data[column].unique()) for column in train_columns]\nsns.set(font_scale=1.2)\naxis = sns.barplot(train_columns, unique_count, log=True)\naxis.set(xlabel='Feature', ylabel='log(Unique_Count)', title='Number of unique values per feature (Train_Sample)')\nfor p_feat, uniq_cnt in zip(axis.patches, unique_count):\n    height = p_feat.get_height()\n    axis.text(p_feat.get_x()+p_feat.get_width()/2,height + 10,uniq_cnt,ha=\"center\") ","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:30.164430Z","iopub.execute_input":"2021-08-10T21:05:30.164718Z","iopub.status.idle":"2021-08-10T21:05:31.766091Z","shell.execute_reply.started":"2021-08-10T21:05:30.164691Z","shell.execute_reply":"2021-08-10T21:05:31.764901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_data[['attributed_time', 'is_attributed']][click_data['is_attributed']==1].describe()\n#Checking whether we have any bad data","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:31.767527Z","iopub.execute_input":"2021-08-10T21:05:31.767877Z","iopub.status.idle":"2021-08-10T21:05:31.793087Z","shell.execute_reply.started":"2021-08-10T21:05:31.767809Z","shell.execute_reply":"2021-08-10T21:05:31.791938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6,6))\nclick_data['is_attributed']=click_data['is_attributed'].astype(int)\nmean = click_data['is_attributed'].mean()\naxis = sns.barplot(['Downloaded (1)', 'Not Downloaded (0)'], [mean, 1-mean])\naxis.set(ylabel='Percentage', title='Downloaded vs Not Downloaded')\nfor p_feat, uniq_cnt in zip(axis.patches, [mean, 1-mean]):\n    height = p_feat.get_height()\n    axis.text(p_feat.get_x()+p_feat.get_width()/2.,height+0.02,'{}%'.format(round(uniq_cnt * 100, 3)),ha=\"center\")","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:31.794422Z","iopub.execute_input":"2021-08-10T21:05:31.794736Z","iopub.status.idle":"2021-08-10T21:05:31.954917Z","shell.execute_reply.started":"2021-08-10T21:05:31.794705Z","shell.execute_reply":"2021-08-10T21:05:31.953714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_data[click_data['is_attributed']==1].ip.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:31.956643Z","iopub.execute_input":"2021-08-10T21:05:31.957051Z","iopub.status.idle":"2021-08-10T21:05:31.982273Z","shell.execute_reply.started":"2021-08-10T21:05:31.957003Z","shell.execute_reply":"2021-08-10T21:05:31.980863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plotting Conversion Rates Vs Most Popular Features","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:31.983726Z","iopub.execute_input":"2021-08-10T21:05:31.984334Z","iopub.status.idle":"2021-08-10T21:05:31.988485Z","shell.execute_reply.started":"2021-08-10T21:05:31.984285Z","shell.execute_reply":"2021-08-10T21:05:31.987443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Conversion Rates Vs Most Popular Apps\nplt.figure(figsize=(25, 20))\ncounts = click_data[['app', 'is_attributed']].groupby('app', as_index=False).count()\ncounts = counts.sort_values('is_attributed', ascending=False)\npercentage = click_data[['app', 'is_attributed']].groupby('app', as_index=False).mean()\npercentage = percentage.sort_values('is_attributed', ascending=False)\n\nmerge = counts.merge(percentage, on='app', how='left')\nmerge.columns = ['app', 'click_cnt', 'conv_percent']\n\naxis = merge[:50].plot(secondary_y='conv_percent')\nplt.title('Conversion Rates Vs 50 Most Popular Apps')\naxis.set(ylabel='Click Count')\nplt.ylabel('% Downloaded')\nplt.show()\ndel counts, percentage, axis\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:31.991519Z","iopub.execute_input":"2021-08-10T21:05:31.992157Z","iopub.status.idle":"2021-08-10T21:05:32.538362Z","shell.execute_reply.started":"2021-08-10T21:05:31.992106Z","shell.execute_reply":"2021-08-10T21:05:32.537466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Conversion Rates Vs Most Popular OS\nplt.figure(figsize=(25, 20))\ncounts = click_data[['os', 'is_attributed']].groupby('os', as_index=False).count()\ncounts = counts.sort_values('is_attributed', ascending=False)\npercentage = click_data[['os', 'is_attributed']].groupby('os', as_index=False).mean()\npercentage = percentage.sort_values('is_attributed', ascending=False)\n\nmerge = counts.merge(percentage, on='os', how='left')\nmerge.columns = ['os', 'click_cnt', 'conv_percent']\n\naxis = merge[:50].plot(secondary_y='conv_percent')\nplt.title('Conversion Rates Vs 50 Most Popular OS')\naxis.set(ylabel='Click Count')\nplt.ylabel('% Downloaded')\nplt.show()\ndel counts, percentage, axis\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:32.539778Z","iopub.execute_input":"2021-08-10T21:05:32.540238Z","iopub.status.idle":"2021-08-10T21:05:33.013207Z","shell.execute_reply.started":"2021-08-10T21:05:32.540201Z","shell.execute_reply":"2021-08-10T21:05:33.011608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Conversion Rates Vs Most Popular Devices\nplt.figure(figsize=(25, 20))\ncounts = click_data[['device', 'is_attributed']].groupby('device', as_index=False).count()\ncounts = counts.sort_values('is_attributed', ascending=False)\npercentage = click_data[['device', 'is_attributed']].groupby('device', as_index=False).mean()\npercentage = percentage.sort_values('is_attributed', ascending=False)\n\nmerge = counts.merge(percentage, on='device', how='left')\nmerge.columns = ['device', 'click_cnt', 'conv_percent']\n\naxis = merge[:50].plot(secondary_y='conv_percent')\nplt.title('Conversion Rates Vs 50 Most Popular Devices')\naxis.set(ylabel='Click Count')\nplt.ylabel('% Downloaded')\nplt.show()\ndel counts, percentage, axis\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:33.014817Z","iopub.execute_input":"2021-08-10T21:05:33.015402Z","iopub.status.idle":"2021-08-10T21:05:33.511911Z","shell.execute_reply.started":"2021-08-10T21:05:33.015332Z","shell.execute_reply":"2021-08-10T21:05:33.510896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Conversion Rates Vs Most Popular IPs\nplt.figure(figsize=(25, 20))\ncounts = click_data[['ip', 'is_attributed']].groupby('ip', as_index=False).count()\ncounts = counts.sort_values('is_attributed', ascending=False)\npercentage = click_data[['ip', 'is_attributed']].groupby('ip', as_index=False).mean()\npercentage = percentage.sort_values('is_attributed', ascending=False)\n\nmerge = counts.merge(percentage, on='ip', how='left')\nmerge.columns = ['ip', 'click_cnt', 'conv_percent']\n\naxis = merge[:200].plot(secondary_y='conv_percent')\nplt.title('Conversion Rates Vs 200 Most Popular IPs')\naxis.set(ylabel='Click Count')\nplt.ylabel('% Downloaded')\nplt.show()\ndel counts, percentage, axis\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:33.513412Z","iopub.execute_input":"2021-08-10T21:05:33.513715Z","iopub.status.idle":"2021-08-10T21:05:34.048546Z","shell.execute_reply.started":"2021-08-10T21:05:33.513685Z","shell.execute_reply":"2021-08-10T21:05:34.047414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Conversion Rates Vs Most Popular Channels\nplt.figure(figsize=(25, 20))\ncounts = click_data[['channel', 'is_attributed']].groupby('channel', as_index=False).count()\ncounts = counts.sort_values('is_attributed', ascending=False)\npercentage = click_data[['channel', 'is_attributed']].groupby('channel', as_index=False).mean()\npercentage = percentage.sort_values('is_attributed', ascending=False)\n\nmerge = counts.merge(percentage, on='channel', how='left')\nmerge.columns = ['channel', 'click_cnt', 'conv_percent']\n\naxis = merge[:50].plot(secondary_y='conv_percent')\nplt.title('Conversion Rates Vs 50 Most Popular Channels')\naxis.set(ylabel='Click Count')\nplt.ylabel('% Downloaded')\nplt.show()\ndel counts, percentage, axis\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:34.049979Z","iopub.execute_input":"2021-08-10T21:05:34.050336Z","iopub.status.idle":"2021-08-10T21:05:34.528695Z","shell.execute_reply.started":"2021-08-10T21:05:34.050299Z","shell.execute_reply":"2021-08-10T21:05:34.527416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let us check if we can see any correlations with time\nplt.figure(figsize=(25, 20))\nclick_data['click_weekday']=click_data['click_time'].dt.day_name()\nclick_data['click_hr']=click_data['click_time'].dt.hour\nclick_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:34.530609Z","iopub.execute_input":"2021-08-10T21:05:34.531094Z","iopub.status.idle":"2021-08-10T21:05:34.995838Z","shell.execute_reply.started":"2021-08-10T21:05:34.531038Z","shell.execute_reply":"2021-08-10T21:05:34.994855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_data['click_weekday'].describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:34.997249Z","iopub.execute_input":"2021-08-10T21:05:34.997530Z","iopub.status.idle":"2021-08-10T21:05:35.162283Z","shell.execute_reply.started":"2021-08-10T21:05:34.997502Z","shell.execute_reply":"2021-08-10T21:05:35.160798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_data['click_hr'].describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:35.163675Z","iopub.execute_input":"2021-08-10T21:05:35.164095Z","iopub.status.idle":"2021-08-10T21:05:35.211775Z","shell.execute_reply.started":"2021-08-10T21:05:35.164045Z","shell.execute_reply":"2021-08-10T21:05:35.210519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Weekday Vs Click count and Conversion Rate\ntemp = click_data[['click_weekday','is_attributed']].groupby(['click_weekday'], as_index=False).mean()\nx = temp['click_weekday']\ny_mean = temp['is_attributed']\ntemp = click_data[['click_weekday','is_attributed']].groupby(['click_weekday'], as_index=False).count()\ny_count = temp['is_attributed']\nprint(temp)\n\nplot = plt.figure()\nsub_plot = plot.add_subplot(111)\naddon = sub_plot.twinx()\n\nsub_plot.set_xlabel(\"Weekday\")\nsub_plot.set_ylabel(\"% Conversions\")\naddon.set_ylabel(\"Count of Clicks\")\n\nplot1, = sub_plot.plot(x, y_mean, color=\"#75a1a6\",label=\"% Conversions\")\nplot2, = addon.plot(x, y_count, color=\"#a675a1\", label=\"Count of Clicks\")\nlines = [plot1, plot2]\nsub_plot.legend(handles=lines, loc='best')\n\nsub_plot.yaxis.label.set_color(\"#75a1a6\")\naddon.yaxis.label.set_color(\"#a675a1\")\n\ndel temp\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:35.213207Z","iopub.execute_input":"2021-08-10T21:05:35.213500Z","iopub.status.idle":"2021-08-10T21:05:35.799977Z","shell.execute_reply.started":"2021-08-10T21:05:35.213469Z","shell.execute_reply":"2021-08-10T21:05:35.798714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Click Hour Vs Click count and Conversion Rate\ntemp = click_data[['click_hr','is_attributed']].groupby(['click_hr'], as_index=False).mean()\nx = temp['click_hr']\ny_mean = temp['is_attributed']\ntemp = click_data[['click_hr','is_attributed']].groupby(['click_hr'], as_index=False).count()\ny_count = temp['is_attributed']\n\nplot = plt.figure()\nsub_plot = plot.add_subplot(111)\naddon = sub_plot.twinx()\n\nsub_plot.set_xlabel(\"Hour of the day\")\nsub_plot.set_ylabel(\"% Conversions\")\naddon.set_ylabel(\"Count of Clicks\")\n\nplot1, = sub_plot.plot(x, y_mean, color=\"#75a1a6\",label=\"% Conversions\")\nplot2, = addon.plot(x, y_count, color=\"#a675a1\", label=\"Count of Clicks\")\nlines = [plot1, plot2]\nsub_plot.legend(handles=lines, loc='best')\n\nsub_plot.yaxis.label.set_color(\"#75a1a6\")\naddon.yaxis.label.set_color(\"#a675a1\")\n\ndel temp\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:35.801295Z","iopub.execute_input":"2021-08-10T21:05:35.801612Z","iopub.status.idle":"2021-08-10T21:05:36.303713Z","shell.execute_reply.started":"2021-08-10T21:05:35.801575Z","shell.execute_reply":"2021-08-10T21:05:36.302797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"click_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-10T21:05:36.304899Z","iopub.execute_input":"2021-08-10T21:05:36.305179Z","iopub.status.idle":"2021-08-10T21:05:36.322142Z","shell.execute_reply.started":"2021-08-10T21:05:36.305140Z","shell.execute_reply":"2021-08-10T21:05:36.321255Z"},"trusted":true},"execution_count":null,"outputs":[]}]}