{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import subprocess\nprint('# Line count:')\nfor file in ['train.csv', 'test.csv', 'train_sample.csv']:\n    lines = subprocess.run(['wc', '-l', '/kaggle/input/talkingdata-adtracking-fraud-detection/{}'.format(file)], stdout=subprocess.PIPE).stdout.decode('utf-8')\n    print(lines, end='', flush=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\npal = sns.color_palette()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/talkingdata-adtracking-fraud-detection/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df = pd.read_csv(path+\"train_sample.csv\", dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'], parse_dates=['click_time'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 8))\ncols = ['ip', 'app', 'device', 'os', 'channel']\nuniques = [len(train_sample_df[col].unique()) for col in cols]\nsns.set(font_scale=1.2)\nax = sns.barplot(cols, uniques, palette=pal, log=True)\nax.set(xlabel='Feature', ylabel='unique count', title='Number of unique values per feature')\nfor p, uniq in zip(ax.patches, uniques):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 10,\n            uniq,\n            ha=\"center\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nsns.set(font_scale=1.2)\nax=sns.countplot(train_sample_df['is_attributed']);\nax.set(ylabel='Count of users', title='Count of users with App Downloaded vs Not Downloaded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6,6))\n#sns.set(font_scale=1.2)\nmean = (train_sample_df.is_attributed.values == 1).mean()\nax = sns.barplot(['App Downloaded (1)', 'Not Downloaded (0)'], [mean, 1-mean])\nax.set(ylabel='Percentage of users', title='Percentage of usres with App Downloaded vs Not Downloaded')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical = ['ip', 'app', 'device', 'os', 'channel']\nfor v in categorical:\n    train_sample_df[v] = train_sample_df[v].astype('category')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check if the IP's have too many clicks and do those IP's convert"},{"metadata":{"trusted":true},"cell_type":"code","source":"#table to see ips with their associated total clicks\nip_repeat_df = train_sample_df['ip'].value_counts().reset_index(name='count_clicks')\nip_repeat_df.columns = ['ip', 'count_clicks']\nip_repeat_df[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df= train_sample_df.merge(ip_repeat_df, on='ip', how='left')\ntrain_sample_df[train_sample_df['is_attributed']==1].sort_values('count_clicks', ascending=False)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df[train_sample_df['is_attributed']==1].ip.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That means IP's with higher number of clicks do install the app and it is not realted to any click spam. However, this anlysis is just based on the sample traiing data which is merely 100,000 records."},{"metadata":{},"cell_type":"markdown","source":"Verify the time trends of the clicks and installs"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df['click_hour']=train_sample_df['click_time'].dt.hour\ntrain_sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Click and conversion trends based on the click hour of the day"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).count().plot()\nplt.title('HOURLY CLICK FREQUENCY Lineplot');\nplt.ylabel('Number of Clicks');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).mean().plot()\nplt.title('HOURLY CLICK FREQUENCY Lineplot');\nplt.ylabel('Number of Clicks');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The trend of click to conversion with hour of the day does not show any clear patterns because during the hours between 15-19, clciks were minimum whereas the conversions were maximum. On the contrarat during the 20th hour, clicks and the corresponding conversions were almost negligible as compared to rets of the day. \nI will look at the actual training data to make some conclusions for the hourly trend and using it a s afeature to train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sample_df['click_DAY']=train_sample_df['click_time'].dt.day\ntrain_sample_df[['click_DAY','is_attributed']].groupby(['click_DAY'], as_index=True).count().plot(kind='bar', color='blue')\nplt.title('Daily CLICK FREQUENCY BARPLOT');\nplt.ylabel('Number of Clicks')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(path+\"train.csv\",  nrows=30000000, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(path+\"test.csv\", dtype=dtypes,skiprows=range(1,11290470), nrows=7500000, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.head())\nprint(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"variables = ['ip', 'app', 'device', 'os', 'channel']\nfor v in variables:\n    train_df[v] = train_df[v].astype('category')\n    test_df[v]=test_df[v].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['click_time'] = pd.to_datetime(train_df['click_time'])\ntest_df['click_time'] = pd.to_datetime(test_df['click_time'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['click_hour']=train_df['click_time'].dt.hour\ntest_df['click_hour']=test_df['click_time'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#check for hourly patterns of training data\ntrain_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).count().plot(kind='bar', color='blue')\nplt.title('HOURLY CLICK FREQUENCY BARPLOT');\nplt.ylabel('Number of Clicks')\n\ntrain_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).mean().plot(kind='bar', color='green')\nplt.title('HOURLY CLICK FREQUENCY BARPLOT');\nplt.ylabel('Number of Clicks')\n\ntrain_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).count().plot(color='blue')\nplt.title('HOURLY CLICK FREQUENCY LINEPLOT');\nplt.ylabel('Number of Clicks');\n\ntrain_df[['click_hour','is_attributed']].groupby(['click_hour'], as_index=True).mean().plot(color='green')\nplt.title('HOURLY CONVERSION RATIO LINEPLOT');\nplt.ylabel('Converted Ratio');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['click_DAY']=train_df['click_time'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[['click_DAY','is_attributed']].groupby(['click_DAY'], as_index=True).count().plot(kind='bar', color='blue')\nplt.title('Daily CLICK FREQUENCY BARPLOT');\nplt.ylabel('Number of Clicks')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The above plot shows that the training data that I have chosen have clicks from two days and all hours of the day does not have data. I am not considering day as the feature. Other thing I can try is to take the hour information because early hours of the morning from 1-3 shows maximum clicks which is a bit odd. "},{"metadata":{},"cell_type":"markdown","source":"Generating new features from IP, app, OS,Device, Channel utilizing the click_hour data alongwith.\nFirst of all, I will combine both training and test data so that all the features can be derived simultaneously for both training and test set."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_samples = len(train_df)\ntrain_df=train_df.append(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Getting the count of clicks for ip-hour combination\n2. Getting the count of clicks for ip-app combination\n3. Getting the count of clicks for ip-app-os combination\n4. Getting the count of clicks for ip-device-os combination\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['ip','click_hour','channel']].groupby(by=['ip','click_hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_hour_count'})\ntrain_df = train_df.merge(temp_df, on=['ip','click_hour'], how='left')\ndel temp_df\ngc.collect()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\ntrain_df = train_df.merge(temp_df, on=['ip','app'], how='left')\ndel temp_df\ngc.collect()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['ip', 'app','os', 'channel']].groupby(by=['ip', 'app','os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\ntrain_df = train_df.merge(temp_df, on=['ip','app','os'], how='left')\ndel temp_df\ngc.collect()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['ip', 'device','os', 'channel']].groupby(by=['ip', 'device','os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_device_os_count'})\ntrain_df = train_df.merge(temp_df, on=['ip','device','os'], how='left')\ndel temp_df\ngc.collect()\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\ndef lgb_modelfit(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',\n                 feval=None, early_stopping_rounds=20, num_boost_round=3000, verbose_eval=10, categorical_features=None):\n    lgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': objective,\n        'metric':metrics,\n        'learning_rate': 0.01,\n        'num_leaves': 31,  \n        'max_depth': -1,  # -1 means no limit\n        'min_child_samples': 20,  \n        'max_bin': 255,  \n        'subsample': 0.6,  \n        'subsample_freq': 0, \n        'colsample_bytree': 0.3,  \n        'min_child_weight': 5,  \n        'subsample_for_bin': 200000,  \n        'min_split_gain': 0,  \n        'reg_alpha': 0,  \n        'reg_lambda': 0,  \n        'nthread': 4,\n        'verbose': 0,\n        'metric':metrics\n    }\n\n    lgb_params.update(params)\n\n    print(\"preparing validation datasets\")\n\n    xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n    xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n\n    evals_results = {}\n\n    bst1 = lgb.train(lgb_params, \n                     xgtrain, \n                     valid_sets=[xgtrain, xgvalid], \n                     valid_names=['train','valid'], \n                     evals_result=evals_results, \n                     num_boost_round=num_boost_round,\n                     early_stopping_rounds=early_stopping_rounds,\n                     verbose_eval=10, \n                     feval=feval)\n\n    n_estimators = bst1.best_iteration\n    print(\"\\nModel Report\")\n    print(\"n_estimators : \", n_estimators)\n    print(metrics+\":\", evals_results['valid'][metrics][n_estimators-1])\n\n    return bst1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = train_df[train_samples:]\nval_df = train_df[(train_samples-3000000):train_samples]\ntrain_df = train_df[:(train_samples-3000000)]\n\nprint(\"train size: \", len(train_df))\nprint(\"valid size: \", len(val_df))\nprint(\"test size : \", len(test_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removed IP from features as the ip has been mainly used to extract new features. \ntarget = 'is_attributed'\npredictors = ['app','device','os', 'channel', 'click_hour',  \n              'ip_hour_count', 'ip_app_count', 'ip_app_os_count','ip_device_os_count' ]\ncategorical = ['app', 'device', 'os', 'channel', 'click_hour']\n\ntest_reference = pd.DataFrame()\ntest_reference['click_id'] = test_df['click_id'].astype('int')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nprint(\"Training...\")\nstart_time = time.time()\n\n\nparams = {\n    'learning_rate': 0.15,\n    'num_leaves': 7,  \n    'max_depth': 3, \n    'min_child_samples': 100,  \n    'max_bin': 100, \n    'subsample': 0.7,  \n    'subsample_freq': 1,  \n    'colsample_bytree': 0.9, \n    'min_child_weight': 0,  \n    'scale_pos_weight':99 # because training data is extremely unbalanced \n}\nbst = lgb_modelfit(params, \n                        train_df, \n                        val_df, \n                        predictors, \n                        target, \n                        objective='binary', \n                        metrics='auc',\n                        early_stopping_rounds=30, \n                        verbose_eval=True, \n                        num_boost_round=500, \n                        categorical_features=categorical)\n\nprint('[{}]: model training time'.format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Calculating predictions\")\ntest_reference['is_attributed'] = bst.predict(test_df[predictors])\nprint(\"writing the results to test_predictions.csv\")\ntest_reference.to_csv('test_predictions.csv',index=False)\nprint(\"Predictions calculated and written into csv file\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_reference.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6,6))\n#sns.set(font_scale=1.2)\nmean = (test_reference.is_attributed.values == 1).mean()\nax = sns.barplot(['App Downloaded (1)', 'Not Downloaded (0)'], [mean, 1-mean])\nax.set(ylabel='Percentage of users', title='Percentage of users with App Downloaded vs Not Downloaded')\nfor p, uniq in zip(ax.patches, [mean, 1-mean]):\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height+0.01,\n            '{}%'.format(round(uniq * 100, 2)),\n            ha=\"center\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb.plot_importance(bst, importance_type='split')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}