{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nThis version has improvements based on new feature engg techniques observed from different kernels. Below are few of them:\n- https://www.kaggle.com/graf10a/lightgbm-lb-0-9675\n- https://www.kaggle.com/rteja1113/lightgbm-with-count-features?scriptVersionId=2815638\n- https://www.kaggle.com/nuhsikander/lgbm-new-features-corrected?scriptVersionId=2852561\n- https://www.kaggle.com/aloisiodn/lgbm-starter-early-stopping-0-9539 (Original script)\n\"\"\"\n\nimport pandas as pd\nimport time\nimport numpy as np\nfrom sklearn.cross_validation import train_test_split\nimport lightgbm as lgb\nimport gc\n\ndef lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',\n                 feval=None, early_stopping_rounds=20, num_boost_round= 50, verbose_eval=10, categorical_features=None):\n    lgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': objective,\n        'metric':metrics,\n        'learning_rate': 0.01,\n        #'is_unbalance': 'true',  #because training data is unbalance (replaced with scale_pos_weight)\n        'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)\n        'max_depth': -1,  # -1 means no limit\n        'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 255,  # Number of bucketed bin for feature values\n        'subsample': 0.6,  # Subsample ratio of the training instance.\n        'subsample_freq': 0,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.3,  # Subsample ratio of columns when constructing each tree.\n        'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'subsample_for_bin': 200000,  # Number of samples for constructing bin\n        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n        'reg_alpha': 0,  # L1 regularization term on weights\n        'reg_lambda': 0,  # L2 regularization term on weights\n        'nthread': 4,\n        'verbose': 0,\n        'metric':metrics\n    }\n\n    lgb_params.update(params)\n\n    print(\"preparing validation datasets\")\n\n    xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n    xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n\n    evals_results = {}\n\n    bst1 = lgb.train(lgb_params, \n                     xgtrain, \n                     valid_sets=[xgtrain, xgvalid], \n                     valid_names=['train','valid'], \n                     evals_result=evals_results, \n                     num_boost_round=num_boost_round,\n                     early_stopping_rounds=early_stopping_rounds,\n                     verbose_eval=10, \n                     feval=feval)\n\n    n_estimators = bst1.best_iteration\n    print(\"\\nModel Report\")\n    print(\"n_estimators : \", n_estimators)\n    print(metrics+\":\", evals_results['valid'][metrics][n_estimators-1])\n\n    return bst1\n\npath = '../input/'\n\ndtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\n\nprint('loading train data...')\ntrain_df = pd.read_csv(path+\"train.csv\", skiprows=range(1,144903891), nrows=40000000, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])\n\nprint('loading test data...')\ntest_df = pd.read_csv(path+\"test.csv\", dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n\nlen_train = len(train_df)\ntrain_df=train_df.append(test_df)\n\ndel test_df\ngc.collect()\n\nprint('Extracting new features...')\ntrain_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('uint8')\ntrain_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('uint8')\n\ngc.collect()\n\nprint('grouping by ip-day-hour combination...')\ngp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_tcount'})\ntrain_df = train_df.merge(gp, on=['ip','day','hour'], how='left')\ndel gp\ngc.collect()\n\nprint('grouping by ip-app combination...')\ngp = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\ntrain_df = train_df.merge(gp, on=['ip','app'], how='left')\ndel gp\ngc.collect()\n\n\nprint('grouping by ip-app-os combination...')\ngp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\ntrain_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')\ndel gp\ngc.collect()\n\n\n# Adding features with var and mean hour (inspired from nuhsikander's script)\nprint('grouping by : ip_day_chl_var_hour')\ngp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})\ntrain_df = train_df.merge(gp, on=['ip','day','channel'], how='left')\ndel gp\ngc.collect()\n\nprint('grouping by : ip_app_os_var_hour')\ngp = train_df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})\ntrain_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')\ndel gp\ngc.collect()\n\nprint('grouping by : ip_app_channel_var_day')\ngp = train_df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})\ntrain_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')\ndel gp\ngc.collect()\n\nprint('grouping by : ip_app_chl_mean_hour')\ngp = train_df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})\nprint(\"merging...\")\ntrain_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')\ndel gp\ngc.collect()\n\nprint(\"vars and data type: \")\ntrain_df.info()\ntrain_df['ip_tcount'] = train_df['ip_tcount'].astype('uint16')\ntrain_df['ip_app_count'] = train_df['ip_app_count'].astype('uint16')\ntrain_df['ip_app_os_count'] = train_df['ip_app_os_count'].astype('uint16')\n\n\ntest_df = train_df[len_train:]\nval_df = train_df[(len_train-2500000):len_train]\ntrain_df = train_df[:(len_train-2500000)]\n\nprint(\"train size: \", len(train_df))\nprint(\"valid size: \", len(val_df))\nprint(\"test size : \", len(test_df))\n\ntarget = 'is_attributed'\npredictors = ['app','device','os', 'channel', 'hour', 'day', \n              'ip_tcount', 'ip_tchan_count', 'ip_app_count',\n              'ip_app_os_count', 'ip_app_os_var',\n              'ip_app_channel_var_day','ip_app_channel_mean_hour']\ncategorical = ['app', 'device', 'os', 'channel', 'hour', 'day']\n\nsub = pd.DataFrame()\nsub['click_id'] = test_df['click_id'].astype('int')\n\ngc.collect()\n\nprint(\"Training...\")\nstart_time = time.time()\n\n\nparams = {\n    'learning_rate': 0.15,\n    #'is_unbalance': 'true', # replaced with scale_pos_weight argument\n    'num_leaves': 7,  # 2^max_depth - 1\n    'max_depth': 3,  # -1 means no limit\n    'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n    'max_bin': 100,  # Number of bucketed bin for feature values\n    'subsample': 0.7,  # Subsample ratio of the training instance.\n    'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n    'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n    'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n    'scale_pos_weight':99 # because training data is extremely unbalanced \n}\nbst = lgb_modelfit_nocv(params, \n                        train_df, \n                        val_df, \n                        predictors, \n                        target, \n                        objective='binary', \n                        metrics='auc',\n                        early_stopping_rounds=30, \n                        verbose_eval=True, \n                        num_boost_round=500, \n                        categorical_features=categorical)\n\nprint('[{}]: model training time'.format(time.time() - start_time))\ndel train_df\ndel val_df\ngc.collect()\n\n","metadata":{"_uuid":"a7adc8e9-2479-47d2-89d7-b5290991cdf0","_cell_guid":"1e2e696c-d744-4ac6-8da3-a25c085bf58e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-10T11:37:16.80848Z","iopub.execute_input":"2021-08-10T11:37:16.808827Z","iopub.status.idle":"2021-08-10T11:37:42.198254Z","shell.execute_reply.started":"2021-08-10T11:37:16.80877Z","shell.execute_reply":"2021-08-10T11:37:42.196804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Predicting...\")\nsub['is_attributed'] = bst.predict(test_df[predictors])\nprint(\"writing...\")\nsub.to_csv('sub_lgb_balanced99.csv',index=False)\nprint(\"done...\")","metadata":{},"execution_count":null,"outputs":[]}]}