{"cells":[{"metadata":{"_uuid":"ad12431d0724a4ed970ec14d7b23f38b05121fe9","_cell_guid":"29288a60-4d93-48ef-ac1a-943c7f6b9538"},"cell_type":"markdown","source":"\nTHANK YOU AND ACKNOLEDGEMENTS:\nThis kernel develops further the ideas suggested in:\n  *  \"lgbm starter - early stopping 0.9539\" by [Aloisio Dourado](https://www.kaggle.com/aloisiodn/lgbm-starter-early-stopping-0-9539/code), \n  * \"LightGBM (Fixing unbalanced data)\" by [Pranav Pandya](https://www.kaggle.com/pranav84/lightgbm-fixing-unbalanced-data-auc-0-9787?scriptVersionId=2777211), \n  * \"LightGBM with count features\" by [Ravi Teja Gutta](https://www.kaggle.com/rteja1113/lightgbm-with-count-features), \n  * \"Try Pranav's R LGBM in Python\" by  [Andy Harless ](https://www.kaggle.com/aharless/try-pranav-s-r-lgbm-in-python/code)\n  \nI would like to extend my gratitude to these individuals for sharing their work.\n\nWHAT IS NEW IN THIS VERSION? \nIn addition to some cosmetic changes to the code/LightGBM parameters, I have added some new features placed in a function all feature engineering part.\n"},{"metadata":{"collapsed":true,"_cell_guid":"4e83f953-f01b-42d7-8e75-7738db1b1e4a","_uuid":"d816d95909bf541b32133282664ec21ec143e8d5","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nimport gc","execution_count":1,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"c0be34be-c8c2-4c98-8032-e627623cdbe9","_uuid":"3b9369ad2044602e1bb5e7a5a69b2edaf9577f54","trusted":true},"cell_type":"code","source":"import os\ncwd = os.getcwd()\ntrain_path = '../input/'+'train.csv'\ntest_path = '../input/'+'test.csv'","execution_count":2,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"21190ed7-fc7d-4bbf-adec-6c00fea0942a","_uuid":"72908fdf9e4f9435241368b83b42c6abb9ed17bf","trusted":true},"cell_type":"code","source":"train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\ntest_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']","execution_count":3,"outputs":[]},{"metadata":{"_uuid":"75d19bd9ba594c56979f48b0b821359b7c0dce19","_cell_guid":"8e8130af-3b7f-4983-bebb-1c971303f3d1","trusted":true},"cell_type":"code","source":"dtypes = {\n        'ip'            : 'uint32',\n        'app'           : 'uint16',\n        'device'        : 'uint16',\n        'os'            : 'uint16',\n        'channel'       : 'uint16',\n        'is_attributed' : 'uint8',\n        'click_id'      : 'uint32'\n        }\nprint('Loading the training data...')\ntrain = pd.read_csv(train_path, skiprows=range(1,84903891), dtype=dtypes, nrows=11000000,usecols=train_cols)","execution_count":4,"outputs":[]},{"metadata":{"_uuid":"f93aeb45b8c8189588560eef458695ce382857e7","_cell_guid":"7b74394c-15af-4fb8-85c0-37db0d6461ce","trusted":true},"cell_type":"code","source":"train.info()","execution_count":5,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"890df849-52f8-4e1b-81fe-0e003ab7bc9d","_uuid":"e80a0f192f7c20e95fa86bf5527e633fb5bd4d45","trusted":false},"cell_type":"code","source":"len_train = len(train)\nprint('The initial size of the train set is', len_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4cb821795d200a91e4abd053b8585141674b663b","_cell_guid":"6cbe84bb-c1c3-43f6-b0b4-dfa3457f959c"},"cell_type":"markdown","source":"## Feature Engineering\n\nSome new feature has been created in relation to clicks over different other features."},{"metadata":{"collapsed":true,"_cell_guid":"1e41e713-af94-4cf7-856a-7e48d8684a17","_uuid":"c73d607778e0f913cde8a38a5dc6a9d6eea77d9d","trusted":false},"cell_type":"code","source":"def df_featured(df):\n    print(\"Creating new time features: 'hour' and 'day'...\")\n    df['hour'] = pd.to_datetime(df.click_time).dt.hour.astype('uint8')\n    df['day'] = pd.to_datetime(df.click_time).dt.day.astype('uint8')\n    gc.collect()\n    \n    print(\"Feature Engineering \\n\")\n    \n    print('1. Computing the number of clicks associated with a given IP address within each hour... ')\n    n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',\n          'hour'])[['channel']].count().reset_index().rename(columns={'channel': 'n_channels'})\n    print('Merging the channels data with the main data set...\\n')\n    df = df.merge(n_channel, on=['ip','day','hour'], how='left')\n    del n_channel\n    gc.collect()\n          \n          \n    print('2. Computing the number of clicks associated with a given IP address and app...')\n    n_channel = df[['ip','app', 'channel']].groupby(by=['ip', \n          'app'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_count'})\n    print('Merging the channels data with the main data set...\\n')\n    df = df.merge(n_channel, on=['ip','app'], how='left')\n    del n_channel\n    gc.collect()\n          \n    print('3. Computing the number of clicks associated with a given IP address, app, and os...')\n    n_channel = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', \n          'os'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_os_count'}) \n    print('Merging the channels data with the main data set...\\n')       \n    df = df.merge(n_channel, on=['ip','app', 'os'], how='left')\n    del n_channel\n    gc.collect()\n          \n    # Adding features with var and mean hour (inspired from nuhsikander's script)  \n          \n    print('4. grouping by : ip_day_chl_var_hour..')\n    n_channel = df[['ip','day','hour','channel']].groupby(by=['ip','day',\n            'channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})\n    print('Merging the hour data with the main data set...\\n') \n    df = df.merge(n_channel, on=['ip','day','channel'], how='left')\n    del n_channel\n    gc.collect()\n          \n    print('5. grouping by : ip_app_os_var_hour..')\n    n_channel = df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', \n                'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})\n    print('Merging the hour data with the main data set...\\n') \n    df = df.merge(n_channel, on=['ip','app', 'os'], how='left')\n    del n_channel\n    gc.collect()\n          \n    print('6. grouping by : ip_app_channel_var_day...')\n    n_channel = df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', \n                'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})\n    print('Merging the day data with the main data set...\\n') \n    df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')\n    del n_channel\n    gc.collect()\n          \n    print('7. grouping by : ip_app_chl_mean_hour..')\n    n_channel = df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', \n                'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})\n    print('Merging the meanhour data with the main data set...\\n') \n    df = df.merge(n_channel, on=['ip','app', 'channel'], how='left')\n    del n_channel\n    gc.collect()\n    \n    print('8. group by : ip_hh_dev')\n    n_channel = df[['ip', 'device', 'hour', 'day', 'channel']].groupby(by=['ip', 'device', 'day',\n             'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_dev'})\n    print('Merging the channel data with the main data set...\\n') \n    df = df.merge(n_channel, on=['ip','device','day','hour'], how='left')\n    del n_channel\n    gc.collect()     \n    df['n_channels'] = df['n_channels'].astype('uint16')\n    df['ip_app_count'] = df['ip_app_count'].astype('uint16')\n    df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint16')\n    df['ip_tchan_count'] = df['ip_tchan_count'].astype('float32')\n    df['ip_app_os_var'] = df['ip_app_os_var'].astype('float32')\n    df['ip_app_channel_var_day'] = df['ip_app_channel_var_day'].astype('float32')\n    df['ip_app_channel_mean_hour'] = df['ip_app_channel_mean_hour'].astype('float32')  \n    df['nip_hh_dev'] = df['nip_hh_dev'].astype('uint16') \n    df.drop( ['ip','day'], axis=1, inplace=True )\n    gc.collect()\n    return df     ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"scrolled":false,"_cell_guid":"79cc4b87-0b13-432f-94e9-b141951085bb","_uuid":"992f8ed1bf3b7994d9650f1e5736897d82f8d0ea","trusted":false},"cell_type":"code","source":"print( \"Train info before: \\n\\n\")\nprint( train.info() )\ntrain = df_featured( train )\ngc.collect()\nprint( \"Train info after: \\n\\n\")\nprint( train.info() )","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"158b63fd-953a-485b-95fe-5200b680e8d1","_uuid":"4bccd367a88e75befbe86621389c5c4e26b1a12f","trusted":false},"cell_type":"code","source":"metrics = 'auc'\nlgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': 'binary',\n        'metric':metrics,\n        'learning_rate': 0.05,\n        'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)\n        'max_depth': -1,  # -1 means no limit\n        'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 255,  # Number of bucketed bin for feature values\n        'subsample': 0.7,  # Subsample ratio of the training instance.\n        'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.7,  # Subsample ratio of columns when constructing each tree.\n        'subsample_for_bin': 200000,  # Number of samples for constructing bin\n        'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n        'nthread': 8,\n        'verbose': 0,\n        'scale_pos_weight':99.7, # because training data is extremely unbalanced \n        'metric':metrics\n}","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"73f8f48d-a78e-48b6-8354-5831b6854336","_uuid":"7f3173e25502463b7b69d5797cb18d821cc708ed","trusted":false},"cell_type":"code","source":"target = 'is_attributed'\ntrain[target] = train[target].astype('uint8')\n\npredictors = ['app','device',  'os', 'channel', 'hour', 'n_channels', 'ip_app_count', 'ip_app_os_count',\n             'ip_tchan_count','ip_app_os_var','ip_app_channel_var_day','ip_app_channel_mean_hour','nip_hh_dev']\ncategorical = [ 'app', 'device', 'os', 'channel']\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"bb9b6b02-68ac-4313-b5ee-05965664b9dc","_uuid":"53395749e81c4285decf81d079afaf5dc99eaf90","trusted":false},"cell_type":"code","source":"# print(train.head(5))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"12f0f309-7e87-4ec3-a244-20024323ffdd","_uuid":"f4b6ef7cfa6e026807a51a33afa0b50c7eae1c12","trusted":false},"cell_type":"code","source":"VALIDATE = False\nMAX_ROUNDS = 1000\nEARLY_STOP = 50\nOPT_ROUNDS = 1000\n\nFULL_OUTFILE = 'sub_lgbm1.csv'\nVALID_OUTFILE = 'sub_lgbm_v.csv'\n\n\nif VALIDATE:\n    train, val_df = train_test_split( train, train_size=.95, shuffle=False )\n\n    print(\"\\nTrain Information \", train.info())\n    print(\"\\nVal Information \", val_df.info())\n\n    print(\"train size: \", len(train))\n    print(\"valid size: \", len(val_df))\n    gc.collect()\n\n    print(\"Training...\\n\")\n\n    num_boost_round=MAX_ROUNDS\n    early_stopping_rounds=EARLY_STOP\n\n    xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical\n                          )\n    del train\n    gc.collect()\n\n    xgvalid = lgb.Dataset(val_df[predictors].values, label=val_df[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical\n                          )\n    del val_df\n    gc.collect()\n\n    evals_results = {}\n\n    bst = lgb.train(lgb_params, \n                     xgtrain, \n                     valid_sets= [xgvalid], \n                     valid_names=['valid'], \n                     evals_result=evals_results, \n                     num_boost_round=num_boost_round,\n                     early_stopping_rounds=early_stopping_rounds,\n                     verbose_eval=10, \n                     feval=None)\n\n    n_estimators = bst.best_iteration\n\n    print(\"\\nModel Report\")\n    print(\"n_estimators : \", n_estimators)\n    print(metrics+\":\", evals_results['valid'][metrics][n_estimators-1])\n    \n    outfile = VALID_OUTFILE\n    \n    del xgvalid\n\nelse:\n\n    \n\n    print(\"train size: \", len(train))\n\n    gc.collect()\n\n    print(\"Training...\")\n\n    num_boost_round=OPT_ROUNDS\n\n    xgtrain = lgb.Dataset(train[predictors].values, label=train[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical\n                          )\n    del train\n    gc.collect()\n    print (\"Dataset preparing done\")\n\n    bst = lgb.train(lgb_params, \n                     xgtrain, \n                     num_boost_round=num_boost_round,\n                     verbose_eval=10, \n                     feval=None)\n    print(\"Traing done\")\n    outfile = FULL_OUTFILE\n\ndel xgtrain\ngc.collect()\n\nprint('load test...')\ntest_cols = ['ip','app','device','os', 'channel', 'click_time', 'click_id']\ntest_df = pd.read_csv(test_path, dtype=dtypes, usecols=test_cols)\n\ntest_df = df_featured( test_df )\ngc.collect()\n\nsub = pd.DataFrame()\nsub['click_id'] = test_df['click_id']\n\nprint(\"Predicting...\")\nsub['is_attributed'] = bst.predict(test_df[predictors])\nprint(\"writing...\")\nsub.to_csv(outfile, index=False, float_format='%.9f')\nprint(\"done...\")\nprint(sub.info())","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"e9681e10-4a45-46b1-9149-8c4ac0fe118f","_uuid":"264fa5769b0e016f8c33a4da49dd035d69e9b747","trusted":false},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"e9a482ca-aece-40cb-91ee-48696986da2b","_uuid":"1c9322d121b927456b3525724f1feb646ea94e68","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}