{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport pickle\nimport lightgbm as gbm\nfrom random import random, seed\n\n\n# Model Parameters\nagg_period = '30min'\n# estimation window limit\nest_window_start = '2017-11-08 0:00:00'\nest_window_end   = '2017-11-09 15:59:59'","execution_count":1,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', index_col = 'click_time', \n                    usecols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'], \n                    dtype = {'is_attributed':bool,'app':np.uint16,'device':np.uint16,'os':np.uint16,'channel':np.uint16,'ip':np.uint32}, \n                    parse_dates=['click_time'])\ntrain = train['2017-11-06 16:00:00':'2017-11-09 15:59:59']\ntrain.reset_index(inplace=True)\ntrain['click_time'] = train['click_time'].dt.floor(agg_period)\ngc.collect()\ntrain.info()","execution_count":2,"outputs":[]},{"metadata":{"_uuid":"bcc826a97a9279c8c47012fdd1e93c5db98a0488","_cell_guid":"82fc4d7b-fafe-4f1b-94e0-862274ab8376","trusted":true},"cell_type":"code","source":"supp = pd.read_csv('../input/test_supplement.csv',\n                   index_col = 'click_time',\n                   usecols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id'],\n                   dtype = {'click_id':np.uint32,'app':np.uint16,'device':np.uint16,\n                            'os':np.uint16,'channel':np.uint16,'ip':np.uint32},\n                   parse_dates=['click_time'])\n\nsupp = supp.loc['2017-11-10 04:00:00':'2017-11-10 15:59:59']\nsupp.reset_index(inplace=True)\nsupp['click_time'] = supp['click_time'].dt.floor(agg_period)\ngc.collect()\nsupp.info()","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"73f0d0c4ebe6bbaa7fd0f56e02e4ca2e190b101e","collapsed":true,"_cell_guid":"9cb953e2-fb8b-4047-bde5-cd7cb86a6d25","trusted":true},"cell_type":"code","source":"# Categorical Variables\ncats = ['app','ip','channel','device','os']","execution_count":5,"outputs":[]},{"metadata":{"_uuid":"d9fa05ff8e9a8c6f2e95190dffbaf11949dca23c","_cell_guid":"ff615839-a0d7-4c4f-92bd-7e2f91c05971","trusted":true},"cell_type":"code","source":"freq = dict()\nfreq_total = len(train) + len(supp)\nfor p_ in [('ip',),('channel',),('app',),('os',),('device',),('click_time',),\n          ('ip','click_time'),('channel','click_time'),('app','click_time'),('os','click_time'),('device','click_time'),\n          ('channel','app'),('ip','device'),('ip','os'),('channel','app','click_time'),\n          ('channel','ip'),('app','ip'),('app','os'),('app','device')]:\n    p = tuple(sorted(p_))\n    freq[p] = pd.concat([train.groupby(p).is_attributed.size(),\n                         supp.groupby(p).click_id.size()],axis=1).sum(axis=1).rename('_'.join(p)+'_freq')\n    print(p,'{:,d}'.format(len(freq[p])),end=' ')\n    freq[p] = freq[p][freq[p] > 1]\n    print(' reduced to ','{:,d}'.format(len(freq[p])))\n    gc.collect()","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"a72083f6e47a6a36ce250cb89e5ff4656c0bc83c","_cell_guid":"ee7ffc86-b9c7-4e73-8634-5dd5da462571","trusted":true},"cell_type":"code","source":"top_k = dict()\nk_of = {'app':20,'ip':5}\nz = pd.concat([train[['ip','app']].groupby(['ip','app']).size(),\n               supp[['ip','app']].groupby(['ip','app']).size()],axis=1).sum(axis=1).rename('tot')\nfor p in ['ip','app']:\n    x = z.reset_index(p).sort_values([p,'tot'], ascending=[True,False]).groupby(p,sort=False).head(k_of[p])\n    top_k[p] = x.groupby(p,sort=False).tot.sum().rename(p+'_top_k')\ndel z\ndel x\ngc.collect()","execution_count":7,"outputs":[]},{"metadata":{"_uuid":"e320ffa6c7157040d844eb5603479943f64601f1","collapsed":true,"_cell_guid":"3057a509-5f3b-4fe9-b753-99be0d11e504","trusted":true},"cell_type":"code","source":"g_rate = dict()\nfor p in cats:\n    g_rate[p] = pd.cut(train.groupby(p).is_attributed.mean(),[0,.0018,.005,1],labels=False)\ng_rate['default'] = 1","execution_count":8,"outputs":[]},{"metadata":{"_uuid":"ea145b3d38dacfeb6cda71c44c10559fc7ba1f75","_cell_guid":"42f400c4-7f87-40b7-b028-d9bcf5672caf","trusted":true},"cell_type":"code","source":"train = pd.concat([train[train.is_attributed==True],train[train.is_attributed==False].sample(3 * 10**6)])\ngc.collect()","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"671161dc0f11aa6d76468c7f679cc307addf14dd","_cell_guid":"937b62e5-d71b-4f72-b5fb-79a1441f6560","trusted":true},"cell_type":"code","source":"transform_code = '''\ndef transf(df,verbose=True):\n            \n    for p in freq:\n        if verbose:\n            print(p,pd.Timestamp.now('US/Eastern').strftime('%H:%M'))\n        df = df.join(freq[p],on=p)\n        df[df.columns[-1]].fillna(1,inplace=True)\n    gc.collect()\n\n    for p in top_k:\n        df[p+'_top_k_pct'] = df[p].map(top_k[p]) / df[p+'_freq']\n    gc.collect()\n    \n    if verbose:\n        print('Conditional Probabilities')\n    fmt = '{}_over_{}'\n    for p in freq:\n        p_code = '_'.join(p)\n        if len(p) == 2:\n            for a in p:\n                df[fmt.format(p_code,a)] = df[p_code+'_freq'] / df[a+'_freq']\n        if len(p) == 3:\n            for a in p:\n                df[fmt.format(p_code,a)] = df[p_code+'_freq'] / df[a+'_freq']\n            for b_ in zip(p,p[1:]+p[0:1]):\n                b = tuple(sorted(b_))\n                b_code = '_'.join(b)\n                df[fmt.format(p_code,b_code)] = df[p_code+'_freq'] / df[b_code+'_freq']\n                \n    for p in freq:\n        p_code = '_'.join(p)\n        df[p_code+'_pct'] = df.pop(p_code+'_freq') / freq_total\n        gc.collect()\n        \n    for p in cats:\n        df[p+'_g_rate'] = df[p].map(g_rate[p]).fillna(g_rate['default'])\n    \n    df['tm'] = df.click_time.dt.hour + df.click_time.dt.minute / 60\n    return df\n'''\n\nexec(transform_code)\n\nfoo = train.iloc[-15:].copy()\nfoo = transf(foo,verbose=True)\nfoo.head(10)","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de7065f60e02940fc14faa002559d1032b6400a8"},"cell_type":"code","source":"train = transf(train.copy())\ntrain.set_index('click_time',inplace=True)\ntrain.sort_index(inplace=True)\ntrain.info()","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"76eb8ff9be7ff66b63923defbc45e546c880db25","_cell_guid":"198d545e-2b0e-4661-86a3-44fdbab7356d","trusted":true},"cell_type":"code","source":"# Store Data Transformation Parameters\npickle.dump([top_k,freq,freq_total,g_rate,transform_code],open('td_dicts.pkl','wb'))\n# Make room\ndel top_k\ndel freq\ndel g_rate\ngc.collect()","execution_count":12,"outputs":[]},{"metadata":{"_uuid":"738b9b3be9c3ecfd96af54d6fbac3c54c6a35408","_cell_guid":"a4f2e577-c5a7-4777-a470-05847dbeac9d","trusted":true},"cell_type":"code","source":"predictors = train.columns\ndropped = ['is_attributed','device_g_rate'] #,'device','os','app','channel','ip']\nprint('Dropped Predictors: ',*[p for p in predictors if p in dropped],sep='\\t')\nplist = [p for p in predictors if p not in dropped]","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"feb87e8c931134260572fe3c0ac462dbe984fa15"},"cell_type":"code","source":"# train gbm\nprint('Begin Model Estimation ...')\nevals_result = {}  # record of fit performance\n\ngc.collect()\n\nl_rates = 'lambda i: max(0.1, 0.25 * (0.999 ** i))'\n\nparams = {'boosting_type':'gbdt', 'objective': 'binary', 'metric': 'auc',\n          'num_leaves': 2000, 'min_data_in_leaf': 4250,\n          'bagging_freq':1, 'bagging_fraction':1, 'feature_fraction':.1,\n          'max_depth': 7, 'scale_pos_weight':1  #, 'mc':monotone_constr\n        }\n\nsettings = dict(init_model=None, num_boost_round=1250, verbose_eval=20, early_stopping_rounds=50)\n\ngbm_train = gbm.Dataset(train.loc['2017-11-08 00:00:00':,plist],\n                        train.is_attributed['2017-11-08 00:00:00':]\n                       )#,categorical_feature=cats)\ngbm_eval  = gbm.Dataset(train.loc[:'2017-11-08 00:00:00',plist],\n                        train.is_attributed[:'2017-11-08 00:00:00'],\n                        reference=gbm_train)\ngbm_model = gbm.train(params, gbm_train, valid_sets=[gbm_train,gbm_eval], evals_result=evals_result,\n                      learning_rates=eval(l_rates), **settings)\nprint('\\nTraining Dataset Size = {:,d} / Validation Dataset Size = {:,d}'.format(gbm_train.num_data(),gbm_eval.num_data()))\nprint('Number of Features {}\\n'.format(gbm_train.num_feature()))","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"21aa32dd241cd059980444bd1057e93de0b7c97f"},"cell_type":"code","source":"gc.collect()\n# export model\npickle.dump(gbm_model,open('model.pkl','wb'))\npickle.dump({'cats':cats,'agg_period':agg_period,'plist':plist},open('param.pkl','wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"67795c59184767baef1755f14e120361db55f1ca"},"cell_type":"code","source":"_ = gbm.plot_metric(evals_result,metric='auc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b9f5b07f11e77aaf3166400c2549703ed66951e1"},"cell_type":"code","source":"gbm.plot_importance(gbm_model,figsize=(8,11))\n_ = gbm.plot_importance(gbm_model,importance_type='gain',figsize=(8,11))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b7d8a981ccd2e1004505cff45a2881780de9877c"},"cell_type":"code","source":"m = gbm_model.dump_model()\npickle.dump([m['feature_names'],m['tree_info'][-1]['tree_structure']],open('model_out.pkl','wb'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}