{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"XGBoost is one of the most favourite algorithm for kagglers. In this notebook I will try to implement XGBoost and will try to generate as much as possible meaningfull features. I have achieved 0.9728 LB with only 10 mil row training data. With more resource and increased data this accuracy can be increased."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"FILENO= 1 #To distinguish the output file name.\ndebug=0  #Whethere or not in debuging mode\nimport pandas as pd\nimport time\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nimport gc\nimport matplotlib.pyplot as plt\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"501dd1b2-ed58-4796-9106-34343a72925a","_uuid":"80385801f96460b98aa9dc7ca8d6105d39c5e46e","trusted":false,"collapsed":true},"cell_type":"code","source":"predictors=[]\ndef do_next_Click( df,agg_suffix='nextClick', agg_type='float32'):\n    \n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    # {'groupby': ['ip', 'channel']},\n    # {'groupby': ['ip', 'os']},\n    \n    # V3\n    {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n#     {'groupby': ['ip', 'os', 'device']},\n    {'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df[all_features].groupby(spec[\n            'groupby']).click_time.shift(-1) - df.click_time).dt.seconds.fillna(2**32-1).astype(agg_type)\n        \n        predictors.append(new_feature)\n        gc.collect()\n    return (df)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3589b3b1-f55c-4caf-9322-7b7f57bb0ca9","_uuid":"75c186553152177e71bccc933600c9319eba07fd","trusted":false,"collapsed":true},"cell_type":"code","source":"\ndef do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'):\n\n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    {'groupby': ['ip', 'channel']},\n    {'groupby': ['ip', 'os']},\n    \n    # V3\n    #{'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    #{'groupby': ['ip', 'os', 'device']},\n    #{'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df.click_time - df[all_features].groupby(spec[\n                'groupby']).click_time.shift(+1) ).dt.seconds.fillna(2**32-1).astype(agg_type)\n        \n        predictors.append(new_feature)\n        gc.collect()\n    return (df)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a8af04b0-e8a1-4ade-9a55-74767248e334","_uuid":"53ed28f02b18dea534b165f4a12bfbbf555e9ee5","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_agg( df, group_cols, agg_type='uint8', show_max=False, show_agg=True ):\n    agg_name='{}_agg'.format('_'.join(group_cols))  \n    if show_agg:\n        print( \"\\nAggregating by \", group_cols ,  '... and saved in', agg_name )\n    gp = df[group_cols].groupby(group_cols).size().rename(agg_name).to_frame().reset_index()\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cf139ff8-0346-41c5-b8d4-4e2a7f3a2809","_uuid":"70eadd74fed79906e8cf3230ac5914424de3608e","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_count( df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_count'.format(('_'.join(group_cols)),(counted))  \n    if show_agg:\n        print( \"\\nCounting \", counted, \" by \", group_cols ,  '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].count().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"61f89606-6547-44a2-83c7-8c77fa78a94b","_uuid":"4216f42bea38c8d138aae9dcddb2e41a39817bb3","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_countuniq( df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_countuniq'.format(('_'.join(group_cols)),(counted))  \n    if show_agg:\n        print( \"\\nCounting unqiue \", counted, \" by \", group_cols ,  '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2e0049a6-285e-47d9-914a-69947b001610","_uuid":"914da0ee39e5f8f10dac77f910c844191d34ed2e","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_cumcount( df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_cumcount'.format(('_'.join(group_cols)),(counted)) \n    if show_agg:\n        print( \"\\nCumulative count by \", group_cols , '... and saved in', agg_name  )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()\n    df[agg_name]=gp.values\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fd0f0261-aa0c-4843-9f63-df67d12b7d17","_uuid":"9cea84276b5dfa3f72709eb6b43a59ffd22b5147","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_mean( df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_mean'.format(('_'.join(group_cols)),(counted))  \n    if show_agg:\n        print( \"\\nCalculating mean of \", counted, \" by \", group_cols , '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"849d0c73-e829-4242-86b8-3c4000a056e3","_uuid":"e9d41228093f6b6fb81a0aeabdd323c0ba6a1bf9","collapsed":true,"trusted":false},"cell_type":"code","source":"def do_var( df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_var'.format(('_'.join(group_cols)),(counted)) \n    if show_agg:\n        print( \"\\nCalculating variance of \", counted, \" by \", group_cols , '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"01d5193e-d2ff-4c86-bafa-d193ebbb83f4","_uuid":"19e2d0833ddeda5580b5a059f638f773c086ade6","trusted":false,"collapsed":true},"cell_type":"code","source":"if debug:\n    print('*** debug parameter set: this is a test run for debugging purposes ***')\ndef DO(frm,to,fileno):\n    dtypes = {\n            'ip'            : 'uint32',\n            'app'           : 'uint16',\n            'device'        : 'uint8',\n            'os'            : 'uint8',\n            'channel'       : 'uint16',\n            'is_attributed' : 'uint8',\n            'click_id'      : 'uint32',\n            }\n\n    print('loading train data...',frm,to)\n    train_df = pd.read_csv(\"../input/train.csv\", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])\n\n    print('\\nloading test data...\\n')\n    if debug:\n        test_df = pd.read_csv(\"../input/test.csv\", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n    else:\n        test_df = pd.read_csv(\"../input/test.csv\", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n\n    len_train = len(train_df)\n    train_df=train_df.append(test_df)\n    \n    del test_df\n    gc.collect()\n    \n    print('\\nExtracting new features...\\n')\n    train_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('int8')\n    train_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('int8')\n    train_df['minute'] = pd.to_datetime(train_df.click_time).dt.minute.astype('int8')\n#     train_df['second'] = pd.to_datetime(train_df.click_time).dt.second.astype('int8')\n    \n    train_df = do_next_Click( train_df,agg_suffix='nextClick', agg_type='uint32'  ); gc.collect()\n    train_df = do_prev_Click( train_df,agg_suffix='prevClick', agg_type='uint32'  ); gc.collect()  ## Removed temporarily due RAM sortage. \n    train_df = do_countuniq( train_df, ['ip'], 'channel' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'os' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'hour' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'minute' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'app'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'device'); gc.collect()\n    train_df = do_countuniq( train_df, ['app'], 'channel'); gc.collect()\n#     train_df = do_countuniq( train_df, ['ip', 'day'], 'hour' ); gc.collect()\n#     train_df = do_countuniq( train_df, ['ip', 'app'], 'os'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip', 'device'], 'channel' ); gc.collect()\n    \n    train_df = do_countuniq( train_df, ['ip', 'device', 'os'], 'channel'); gc.collect()\n#     train_df = do_countuniq( train_df, ['ip','day','hour'], 'channel' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip','app', 'os'], 'channel' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip', 'device', 'os'], 'app'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip','app', 'os', 'device'], 'channel' ); gc.collect()\n \n\n    train_df = do_cumcount( train_df, ['ip'], 'os'); gc.collect()\n    train_df = do_cumcount( train_df, ['ip', 'device', 'os'], 'app'); gc.collect()\n    \n#     train_df = do_count( train_df, ['ip','day','hour'], 'channel' ); gc.collect()\n    train_df = do_count( train_df, ['ip','app', 'os'], 'channel' ); gc.collect()\n    train_df = do_count( train_df, ['ip','app', 'os', 'device'], 'channel' ); gc.collect()\n    \n    train_df = do_agg( train_df, ['ip', 'day', 'hour'] ); gc.collect()\n    train_df = do_agg( train_df, ['ip', 'app']); gc.collect()\n    train_df = do_agg( train_df, ['ip', 'app', 'os']); gc.collect()\n#     train_df = do_var( train_df, ['ip', 'day', 'channel'], 'hour'); gc.collect()\n#     train_df = do_var( train_df, ['ip', 'app', 'os'], 'hour'); gc.collect()\n#     train_df = do_var( train_df, ['ip', 'app', 'channel'], 'day'); gc.collect()\n#     train_df = do_mean( train_df, ['ip', 'app', 'channel'], 'hour' ); gc.collect()\n    del train_df['day']\n    gc.collect()\n    \n    del train_df['minute']\n    gc.collect()\n   \n    print('\\n\\nBefore appending predictors...\\n\\n', predictors )\n    print('\\n\\nBefore appending predictors length...', len(predictors) )\n    target = 'is_attributed'\n    word= ['app','device','os', 'channel', 'hour']\n    for feature in word:\n        if feature not in predictors:\n            predictors.append(feature)\n    ##### Removing less important feature as they will change in test set         \n#     for x in ['day','minute']:      \n#         predictors.remove(x) # Day is \n    ################################    \n    predictors_sorted= sorted(list(set(predictors))) # to remove any dublicate items\n  \n    categorical = ['app', 'device', 'os', 'channel', 'hour']\n    print('\\n\\nAfter appending predictors...\\n\\n',predictors_sorted )\n    print('\\nAfter appending predictors length...', len(predictors_sorted) )\n\n    test_df = train_df[len_train:]\n    gc.collect()\n    val_df = train_df[(len_train-val_size):len_train]\n    gc.collect()\n    train_df = train_df[:(len_train-val_size)] \n    gc.collect()\n    print(\"train size: \", len(train_df))\n    print(\"valid size: \", len(val_df))\n    print(\"test size : \", len(test_df))\n\n    sub = pd.DataFrame()\n    sub['click_id'] = test_df['click_id'].astype('int')\n\n    gc.collect()\n\n    print(\"\\n\\nTraining...\")\n    start_time = time.time()\n\n    xgb_params ={'eta': 0.3,\n                'max_depth': 6,\n                'learning_rate': 0.3,\n                'n_estimators': 500,\n                'silent': False,\n                'objective': 'binary:logistic',\n                'eval_metric': 'auc', \n                'nthread':24,\n                 'njobs': -1,\n                'gamma': 5.103973694670875e-08,\n                'max_delta_step': 20,\n                'min_child_weight': 4,\n                'subsample': 0.7,\n                'colsample_bylevel': 0.1,\n                'colsample_bytree': 0.7,\n                'reg_alpha': 1e-09,\n                'reg_lambda': 1000.0,\n                'scale_pos_weight': 499.99999999999994,\n                'random_state': 84,\n                ' tree_method':'approx'\n                } \n    \n    xgtrain = xgb.DMatrix(train_df[predictors_sorted].values, label=train_df[target].values)\n    xgvalid = xgb.DMatrix(val_df[predictors_sorted].values, label=val_df[target].values)\n    del train_df\n    del val_df\n    gc.collect()\n    \n    trained_model = xgb.train(xgb_params,xgtrain, 300,[(xgvalid, 'valid')],\n                              maximize=True, early_stopping_rounds=50,verbose_eval=10)\n\n    xgbtest=xgb.DMatrix(test_df[predictors_sorted].values)\n    print(\"\\nModel Report\")\n    print(\"bst1.best_iteration: \", trained_model.best_iteration)\n    \n    print('[{}]: Training time for  XGB'.format(time.time() - start_time))\n    \n    ax = xgb.plot_importance(trained_model, max_num_features=300)\n    plt.gcf().savefig('test%d.png'%(fileno), dpi=600)\n    plt.show()\n\n    print(\"Predicting...\")\n    sub['is_attributed'] = trained_model.predict(xgbtest,ntree_limit=trained_model.best_ntree_limit)\n#     if not debug:\n#         print(\"writing...\")\n    sub.to_csv('sub_it%d.csv'%(fileno),index=False,float_format='%.9f')\n    print(\"done...\")\n    return sub\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f7d21e58-d2fb-4ede-921d-7bb54257196b","_uuid":"71bc14b7be24f18ea9f1fd643ea6ef831cfd6d02","trusted":false,"collapsed":true},"cell_type":"code","source":"nrows=184903891-1\nnchunk=12000000\nval_size=1000000\n\nfrm=nrows-84903891\nif debug:\n    frm=0\n    nchunk=100000\n    val_size=10000\n\nto=frm+nchunk\n\nsub=DO(frm,to,FILENO)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bed7cfba-3429-4717-80cf-25a88edf3bac","_uuid":"1fa9164d30ed405ab7038bdb226bae12bd20c04f","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}