{"cells":[{"metadata":{"_cell_guid":"5da455c0-d393-4d92-bdf3-4c145666be7f","_uuid":"91bb06a88d03d211c702f27a2d9f573950bf9b36"},"cell_type":"markdown","source":"I made cosmetic changes in the [code](https://www.kaggle.com/aharless/kaggle-runnable-version-of-baris-kanber-s-lightgbm/code). Added some new features. Ran for 25mil chunk rows."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d94f2692-fa57-4918-9b46-d5568be01962","_uuid":"22f00a428d4d51f5b1f4686fbda77675d4d2acfa"},"cell_type":"markdown","source":"Importing required libraries "},{"metadata":{"_cell_guid":"55d9fa3a-2f19-4d23-8c8c-b7def3fb304a","_uuid":"22d621b167f5616378f84429d3d0367c0dc09972","collapsed":true,"trusted":true},"cell_type":"code","source":"FILENO= 23 #To distinguish the output file name3\ndebug=0  #Whethere or not in debuging mode\nimport pandas as pd\nimport time\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nimport gc\nimport matplotlib.pyplot as plt\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1f74236d-1d60-4a50-8533-a7073d19f84b","_uuid":"2360d210ef7c2bb874aa85d2ee017a124c653d8f"},"cell_type":"markdown","source":"## Feature extraction\n"},{"metadata":{"_cell_guid":"8ee298c2-426c-46aa-bdde-fb581c5c8ceb","_uuid":"99b0307d67ee48707278d6a3fccc8a8bf97bbde5"},"cell_type":"markdown","source":"## Extracting next click feature \n    ### Taken help from https://www.kaggle.com/nanomathias/feature-engineering-importance-testing\n    ###Did some Cosmetic changes "},{"metadata":{"_cell_guid":"091fef05-bd0d-40c8-8759-9ef016bd9333","_uuid":"09ad65bef81affca447f56286ad69bd7d51ab26e","collapsed":true,"trusted":true},"cell_type":"code","source":"predictors=[]\ndef do_next_Click( df,agg_suffix='nextClick', agg_type='float32'):\n    \n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    # {'groupby': ['ip', 'channel']},\n    # {'groupby': ['ip', 'os']},\n    \n    # V3\n    {'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    {'groupby': ['ip', 'os', 'device']},\n    {'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df[all_features].groupby(spec[\n            'groupby']).click_time.shift(-1) - df.click_time).dt.seconds.astype(agg_type)\n        \n        predictors.append(new_feature)\n        gc.collect()\n    return (df)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3a115fe7-ed2f-4f35-82ed-2c683062626d","_uuid":"de0a00158c153a810aead9a919c88774a0243e5e","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_prev_Click( df,agg_suffix='prevClick', agg_type='float32'):\n\n    print(f\">> \\nExtracting {agg_suffix} time calculation features...\\n\")\n    \n    GROUP_BY_NEXT_CLICKS = [\n    \n    # V1\n    # {'groupby': ['ip']},\n    # {'groupby': ['ip', 'app']},\n    {'groupby': ['ip', 'channel']},\n    {'groupby': ['ip', 'os']},\n    \n    # V3\n    #{'groupby': ['ip', 'app', 'device', 'os', 'channel']},\n    #{'groupby': ['ip', 'os', 'device']},\n    #{'groupby': ['ip', 'os', 'device', 'app']}\n    ]\n\n    # Calculate the time to next click for each group\n    for spec in GROUP_BY_NEXT_CLICKS:\n    \n       # Name of new feature\n        new_feature = '{}_{}'.format('_'.join(spec['groupby']),agg_suffix)    \n    \n        # Unique list of features to select\n        all_features = spec['groupby'] + ['click_time']\n\n        # Run calculation\n        print(f\">> Grouping by {spec['groupby']}, and saving time to {agg_suffix} in: {new_feature}\")\n        df[new_feature] = (df.click_time - df[all_features].groupby(spec[\n                'groupby']).click_time.shift(+1) ).dt.seconds.astype(agg_type)\n        \n        predictors.append(new_feature)\n        gc.collect()\n    return (df)    \n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"751c0380-7970-47df-be44-baae2425a77c","_uuid":"b01392db42e8d2f9b2629ae99019f0845162cc17"},"cell_type":"markdown","source":"Below a function is written to extract count feature by aggregating different cols"},{"metadata":{"_cell_guid":"a04d3a0b-3d3d-4021-835b-2814d5056a32","_uuid":"743edc09c1db135a6f167e5cd05b35bd49a24dea","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_count( df, group_cols, agg_type='uint32', show_max=False, show_agg=True ):\n    agg_name='{}count'.format('_'.join(group_cols))  \n    if show_agg:\n        print( \"\\nAggregating by \", group_cols ,  '... and saved in', agg_name )\n    gp = df[group_cols][group_cols].groupby(group_cols).size().rename(agg_name).to_frame().reset_index()\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1fa53a6f-ab8c-475b-b85c-bab40ac71b87","_uuid":"a047cc91760825e825e855834e944258f0cab9fb"},"cell_type":"markdown","source":"Below a function is written to extract unique count feature from different cols"},{"metadata":{"_cell_guid":"915402dd-25a6-4e7d-859e-28c03110ddcd","_uuid":"cf95bdcf3a431fe8b41e1727f1f0363d3cc7cb81","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_countuniq( df, group_cols, counted, agg_type='uint32', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_countuniq'.format(('_'.join(group_cols)),(counted))  \n    if show_agg:\n        print( \"\\nCounting unqiue \", counted, \" by \", group_cols ,  '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"843361f4-e83a-4791-b0df-6cf2a1cd913b","_uuid":"e19737c5329fe8af3ff15f91f1d9e5ac4c8fd0eb"},"cell_type":"markdown","source":"Below a function is written to extract cumulative count feature  from different cols"},{"metadata":{"_cell_guid":"58347120-b264-4f0b-b551-7b930b3965de","_uuid":"9963dfaec00c438f0b7ec62570e8bd7a964d5da1","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_cumcount( df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_cumcount'.format(('_'.join(group_cols)),(counted)) \n    if show_agg:\n        print( \"\\nCumulative count by \", group_cols , '... and saved in', agg_name  )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()\n    df[agg_name]=gp.values\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0853b2d2-5e75-439d-a452-7f9f9235377a","_uuid":"7a6ec73378df671d3e6e6e939b3c8ee8640548c4"},"cell_type":"markdown","source":"Below a function is written to extract mean feature  from different cols"},{"metadata":{"_cell_guid":"0def6187-4787-4499-8b8c-5be6f992b978","_uuid":"7b6b5345966d0c6046db3ef1d3e40610f95ffacb","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_mean( df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_mean'.format(('_'.join(group_cols)),(counted))  \n    if show_agg:\n        print( \"\\nCalculating mean of \", counted, \" by \", group_cols , '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e111687a-80bc-47b2-97a7-92d8342f2c53","_uuid":"55e9faf3f0b9badd405ff8fc14b4887b42e35cd4","collapsed":true,"trusted":true},"cell_type":"code","source":"def do_var( df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True ):\n    agg_name= '{}_by_{}_var'.format(('_'.join(group_cols)),(counted)) \n    if show_agg:\n        print( \"\\nCalculating variance of \", counted, \" by \", group_cols , '... and saved in', agg_name )\n    gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    if show_max:\n        print( agg_name + \" max value = \", df[agg_name].max() )\n    df[agg_name] = df[agg_name].astype(agg_type)\n    predictors.append(agg_name)\n#     print('predictors',predictors)\n    gc.collect()\n    return( df )","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a24afbe9-daaa-4428-9660-01f41cdf45ce","_uuid":"bb9a2ba9f2e4af91dbc2a4db2839fcbfedabfdb4"},"cell_type":"markdown","source":"A function is written to train the lightGBM model with different given parameters"},{"metadata":{"_cell_guid":"61b250c4-69a6-4f2c-ab95-43224cac526f","_uuid":"787ef6a23e1f2baa0ddabb730d8bd59e47f57b9a","collapsed":true,"trusted":true},"cell_type":"code","source":"###  A function is written to train the lightGBM model with different given parameters\nif debug:\n    print('*** debug parameter set: this is a test run for debugging purposes ***')\n\ndef lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',\n                 feval=None, early_stopping_rounds=50, num_boost_round=3000, verbose_eval=10, categorical_features=None):\n    lgb_params = {\n        'boosting_type': 'gbdt',\n        'objective': objective,\n        'metric':metrics,\n        'learning_rate': 0.04,\n        #'is_unbalance': 'true',  #because training data is unbalance (replaced with scale_pos_weight)\n        'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)\n        'max_depth': -1,  # -1 means no limit\n        'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 255,  # Number of bucketed bin for feature values\n        'subsample': 0.6,  # Subsample ratio of the training instance.\n        'subsample_freq': 0,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.3,  # Subsample ratio of columns when constructing each tree.\n        'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'subsample_for_bin': 200000,  # Number of samples for constructing bin\n        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n        'reg_alpha': 0.99,  # L1 regularization term on weights\n        'reg_lambda': 0.9,  # L2 regularization term on weights\n        'nthread': 8,\n        'verbose': 1,\n    }\n\n    lgb_params.update(params)\n\n    print(\"preparing validation datasets\")\n\n    xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n    xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,\n                          feature_name=predictors,\n                          categorical_feature=categorical_features\n                          )\n    del dtrain\n    del dvalid\n    gc.collect()\n\n    evals_results = {}\n\n    bst1 = lgb.train(lgb_params, \n                     xgtrain, \n                     valid_sets=[ xgvalid], \n                     valid_names=['valid'], \n                     evals_result=evals_results, \n                     num_boost_round=num_boost_round,\n                     early_stopping_rounds=early_stopping_rounds,\n                     verbose_eval=10, \n                     feval=feval)\n\n    print(\"\\nModel Report\")\n    print(\"bst1.best_iteration: \", bst1.best_iteration)\n    print(metrics+\":\", evals_results['valid'][metrics][bst1.best_iteration-1])\n\n    return (bst1,bst1.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b997c512-9e16-47af-af2b-9edfaa7ecc11","_uuid":"1468bb03fc78c6dce86d8329d4fed02cfad5874e"},"cell_type":"markdown","source":"## Running the full calculation.\nA function is written here to run the full calculation with defined parameters."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"def DO(frm,to,fileno):\n    dtypes = {\n            'ip'            : 'uint32',\n            'app'           : 'uint16',\n            'device'        : 'uint8',\n            'os'            : 'uint16',\n            'channel'       : 'uint16',\n            'is_attributed' : 'uint8',\n            'click_id'      : 'uint32',\n            }\n\n    print('loading train data...',frm,to)\n    train_df = pd.read_csv(\"../input/train.csv\", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])\n\n    print('loading test data...')\n    if debug:\n        test_df = pd.read_csv(\"../input/test.csv\", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n    else:\n        test_df = pd.read_csv(\"../input/test.csv\", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n\n    len_train = len(train_df)\n    train_df=train_df.append(test_df)\n    \n    del test_df\n        \n    gc.collect()\n    train_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('int8')\n    train_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('int8') \n    train_df = do_next_Click( train_df,agg_suffix='nextClick', agg_type='float32'  ); gc.collect()\n    train_df = do_prev_Click( train_df,agg_suffix='prevClick', agg_type='float32'  ); gc.collect()  ## Removed temporarily due RAM sortage. \n    \n    train_df = do_countuniq( train_df, ['ip'], 'channel' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip', 'device', 'os'], 'app'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip', 'day'], 'hour' ); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'app'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip', 'app'], 'os'); gc.collect()\n    train_df = do_countuniq( train_df, ['ip'], 'device'); gc.collect()\n    train_df = do_countuniq( train_df, ['app'], 'channel'); gc.collect()\n    train_df = do_cumcount( train_df, ['ip'], 'os'); gc.collect()\n    train_df = do_cumcount( train_df, ['ip', 'device', 'os'], 'app'); gc.collect()\n    train_df = do_count( train_df, ['ip', 'day', 'hour'] ); gc.collect()\n    train_df = do_count( train_df, ['ip', 'app']); gc.collect()\n    train_df = do_count( train_df, ['ip', 'app', 'os']); gc.collect()\n    # train_df = do_var( train_df, ['ip', 'day', 'channel'], 'hour'); gc.collect()\n    train_df = do_var( train_df, ['ip', 'app', 'os'], 'hour'); gc.collect()\n    # train_df = do_var( train_df, ['ip', 'app', 'channel'], 'day'); gc.collect()\n    # train_df = do_mean( train_df, ['ip', 'app', 'channel'], 'hour' ); gc.collect()\n    \n    del train_df['day']\n    gc.collect()\n    gc.collect()\n    \n    \n    print('\\n\\nBefore appending predictors...\\n\\n',sorted(predictors))\n    target = 'is_attributed'\n    word= ['app','device','os', 'channel', 'hour']\n    for feature in word:\n        if feature not in predictors:\n            predictors.append(feature)\n    categorical = ['app', 'device', 'os', 'channel', 'hour']\n    print('\\n\\nAfter appending predictors...\\n\\n',sorted(predictors))\n\n    test_df = train_df[len_train:]\n    val_df = train_df[(len_train-val_size):len_train]\n    train_df = train_df[:(len_train-val_size)]\n\n    print(\"\\ntrain size: \", len(train_df))\n    print(\"\\nvalid size: \", len(val_df))\n    print(\"\\ntest size : \", len(test_df))\n\n    sub = pd.DataFrame()\n    sub['click_id'] = test_df['click_id'].astype('int')\n\n    gc.collect()\n\n    print(\"Training...\")\n    start_time = time.time()\n\n    params = {\n        'learning_rate': 0.10,\n        #'is_unbalance': 'true', # replaced with scale_pos_weight argument\n        'num_leaves': 7,  # 2^max_depth - 1\n        'max_depth': 3,  # -1 means no limit\n        'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n        'max_bin': 100,  # Number of bucketed bin for feature values\n        'subsample': 0.7,  # Subsample ratio of the training instance.\n        'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n        'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n        'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n        'scale_pos_weight':200 # because training data is extremely unbalanced \n    }\n    (bst,best_iteration) = lgb_modelfit_nocv(params, \n                            train_df, \n                            val_df, \n                            predictors, \n                            target, \n                            objective='binary', \n                            metrics='auc',\n                            early_stopping_rounds=30, \n                            verbose_eval=True, \n                            num_boost_round=1000, \n                            categorical_features=categorical)\n\n    print('[{}]: model training time'.format(time.time() - start_time))\n    del train_df\n    del val_df\n    gc.collect()\n\n\n    ax = lgb.plot_importance(bst, max_num_features=300)\n    \n    plt.savefig('test%d.png'%(fileno), dpi=600, bbox_inches='tight')\n    plt.show()\n\n    print(\"Predicting...\")\n    sub['is_attributed'] = bst.predict(test_df[predictors],num_iteration=best_iteration)\n#     if not debug:\n#         print(\"writing...\")\n    sub.to_csv('sub_it%d.csv'%(fileno),index=False,float_format='%.9f')\n    print(\"done...\")\n    return sub","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c27433ea-22c9-4855-825e-215277520dcd","_uuid":"789d80a1ad52daebaabcd50e9c27a24649c6609b"},"cell_type":"markdown","source":""},{"metadata":{"_cell_guid":"792f8444-2cd1-45f1-93df-b0f9f56140ae","_uuid":"b7522cacc79ccb3a96196e986c8ca94ad4380c31","collapsed":true,"trusted":true},"cell_type":"code","source":"nrows=184903891-1\nnchunk=184903890\nval_size=40000000\n\nfrm=nrows-65000000\nif debug:\n    frm=0\n    nchunk=100000\n    val_size=10000\n\nto=frm+nchunk\n\nsub=DO(frm,to,FILENO)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e2cef358-7586-4feb-8f58-b39bc132f357","_uuid":"e0879ed923bc3ab6199d392a2550955354dae7a8","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}