{"cells":[{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2749e3cc00c916bb550d0b0da3789130e77696b0"},"cell_type":"code","source":"import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\nimport lightgbm as lgb\nimport gc\nimport os\nimport time\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9d0bb61c064876de1daf0fa44c114731a2a636d6"},"cell_type":"code","source":"def log(content, *args):\n    tag = 'TRAINING' if TRAIN else 'PREDICTING'\n    print(time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime()), tag, ': ', content, *args)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"2d9cc63ea93652e57e8c806ba2842ee272c12c66"},"cell_type":"code","source":"def load_data(data_type):\n    log('#1 load data')\n\n    dtypes = {\n        'ip': 'category',\n        'app': 'category',\n        'device': 'category',\n        'os': 'category',\n        'channel': 'category',\n        'is_attributed': 'uint8'\n    }\n    if data_type == 'train':\n        # 1 8000 0000多万条数据，要取最后一天的数据,,\n        df = pd.read_csv('../input/train.csv', parse_dates=['click_time', 'attributed_time'], dtype=dtypes,\n                           skiprows=range(1, 131886953), nrows=1000000)\n\n    elif data_type == 'val1':\n        df = pd.read_csv('../input/train.csv', parse_dates=['click_time', 'attributed_time'], dtype=dtypes,\n                          nrows=1000000)\n\n    elif data_type == 'val2':\n        df = pd.read_csv('../input/train.csv', parse_dates=['click_time', 'attributed_time'], dtype=dtypes,\n                         skiprows=range(1, 80000000), nrows=1000000)\n\n    elif data_type == 'test':\n        df = pd.read_csv('../input/test.csv', parse_dates=['click_time'], dtype=dtypes)\n        df.drop(columns='click_id', inplace=True)\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5ceb93f99b599d86abc43b46d76461db5edaf2a2"},"cell_type":"code","source":"\ndef feature_eng(df):\n    log('#2 feature_eng')\n\n    # assume ip device and os could be regcognized as a user\n    df['ip_device_os'] = df['ip'].str.cat(df['device'], sep=' ').str.cat(df['os'], sep=' ')\n\n    # click count per ip device os\n    click_count = df[['ip_device_os', 'click_time']].groupby('ip_device_os').click_time.count()\n    df['click_count'] = df['ip_device_os'].map(click_count)\n\n    # channel count per ip device os\n    channel_count = df[['ip_device_os', 'channel']].groupby('ip_device_os')['channel'].nunique()\n    df['channel_count'] = df['ip_device_os'].map(channel_count)\n\n    # app count per ip device os\n    app_count = df[['ip_device_os', 'app']].groupby('ip_device_os')['app'].nunique()\n    df['app_count'] = df['ip_device_os'].map(app_count)\n\n    # add hour into data set\n    df['click_hour'] = df['click_time'].dt.hour\n\n    hour_count = df[['ip_device_os', 'click_hour']].groupby('ip_device_os').click_hour.nunique()\n    df['hour_count'] = df['ip_device_os'].map(hour_count)\n\n    # more features\n    # 1. 对于某ip device os设备，每天每个小时内，点击数量\n    # 2. 对于某ip device os设备, 针对某个app的点击数量\n    # 3. 对于某ip device os设备, 针对某channel的点击数量\n    # 4. 对于某ip device os设备， 针对每app，每chanel的点击数量\n\n    # 1. 某设备，某个hour内的点击次数\n    df['ip_device_os_hour'] = df['ip_device_os'].str.cat(df['click_hour'].map(str), sep=' ')\n    click_count_in_hour = df[['ip_device_os_hour', 'click_time']].groupby('ip_device_os_hour').click_time.count()\n    df['click_count_in_hour'] = df['ip_device_os_hour'].map(click_count_in_hour)\n    df.drop(columns='ip_device_os_hour', inplace=True)\n\n    # 2. 某app\n    df['ip_device_os_app'] = df['ip_device_os'].str.cat(df['app'].map(str), sep=' ')\n    click_count_for_app = df[['ip_device_os_app', 'click_time']].groupby('ip_device_os_app').click_time.count()\n    df['click_count_for_app'] = df['ip_device_os_app'].map(click_count_for_app)\n    df.drop(columns='ip_device_os_app', inplace=True)\n\n    # # 3. 某channel, 经检测，效果不理想，没有提升。\n    # df['ip_device_os_channel'] = df['ip_device_os'].str.cat(df['channel'].map(str), sep=' ')\n    # click_count_for_channel = df[['ip_device_os_channel', 'click_time']].groupby(\n    #     'ip_device_os_channel').click_time.count()\n    # df['click_count_for_channel'] = df['ip_device_os_channel'].map(click_count_for_channel)\n    # df.drop(columns='ip_device_os_channel', inplace=True)\n\n    # 5. per ip device os under a certain hour, click count for an app.\n\n    del click_count, channel_count, app_count, click_count_in_hour, click_count_for_app#, click_count_for_channel\n    gc.collect()\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ae51131f2dc5c8e8539dcb197545ebb689bad6ee"},"cell_type":"code","source":"def feature_selection(df):\n    log('#3 feature selection')\n\n    drops = ['ip_device_os', 'ip', 'device', 'os', 'click_time']\n    if TRAIN:\n        drops.append( 'attributed_time')\n\n    df.drop(columns=drops, inplace=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"4dc2e37bb7565c5619c5b7c2acb970baacdbe3d5"},"cell_type":"code","source":"def feature_wrangle(df):\n    log('#4 feature wrangle')\n\n    if TRAIN:\n        df['is_attributed'] = df['is_attributed'].fillna(0).astype('uint8')\n\n    df.channel_count = df.channel_count.astype('uint8')\n    df.app_count = df.app_count.astype('uint8')\n    df.click_count = df.click_count.astype('uint16')\n    df.app = df.app.astype('uint16')\n    df.channel = df.channel.astype('uint16')\n    df.hour_count = df.hour_count.astype('uint8')\n    df.click_hour = df.click_hour.astype('uint8')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"44fa0451edc4d8d4b7e46a5a820e76f7176b93cc"},"cell_type":"code","source":"def get_data(data_type):\n    log('GET DATA BY ', data_type)\n\n    # 1. load data\n    df = load_data(data_type)\n    gc.collect()\n\n    # 2. feature eng\n    df = feature_eng(df)\n    gc.collect()\n\n    # 3. feature selection, drop\n    df = feature_selection(df)\n    gc.collect()\n\n    # 4. feature_wrangle\n    df = feature_wrangle(df)\n    gc.collect()\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ae6f9812ce641365eea0339315b4646fbba5c723"},"cell_type":"code","source":"def train_without_cv(has_multi_val):\n    log('train procedure')\n\n    # 1. get final data\n    df_train = get_data('train')\n    df_val = get_data('val1')\n    if has_multi_val:\n        df_val2 = get_data('val2')\n\n    gc.collect()\n\n    # build data set\n    train_data = lgb.Dataset(df_train.drop(columns='is_attributed'), label=df_train['is_attributed'],\n                             categorical_feature=['app', 'channel', 'click_hour'])\n\n    val_data = lgb.Dataset(df_val.drop(columns='is_attributed'), label=df_val['is_attributed'],\n                           reference=train_data)\n\n    if has_multi_val:\n        val_data2 = lgb.Dataset(df_val2.drop(columns='is_attributed'), label=df_val2['is_attributed'],\n                                reference=train_data)\n\n    del df_train, df_val\n    if has_multi_val:\n        del df_val2\n\n    gc.collect()\n\n    log('start training')\n\n    param = {\n        \"boosting_type\": \"gbdt\",\n        \"objective\": \"binary\",\n        \"metric\": \"auc\",\n        \"learning_rate\": 0.07,\n        \"num_leaves\": 90,\n        \"max_depth\": 7,\n#         \"min_data_in_leaf\": 500,\n        \"subsample\": 0.8,\n        \"subsample_freq\": 1,\n        \"feature_fraction\": 0.8,\n        # \"min_child_weight\": 0,\n        # \"subsample_for_bin\": 1000000,\n        # \"min_split_gain\": 0,\n#         'max_bin': 200,\n#         'min_data_in_bin': 3,\n        \"reg_lambda\": 1,\n        'scale_pos_weight': 99.8\n    }\n\n    if not has_multi_val:\n        bst = lgb.train(param, train_data, num_boost_round=200, valid_sets=[train_data, val_data],\n                        valid_names=['train', 'val'], verbose_eval=5, early_stopping_rounds=30)\n    else:\n        bst = lgb.train(param, train_data, num_boost_round=200, valid_sets=[train_data, val_data, val_data2],\n                        valid_names=['train', 'val', 'val2'], verbose_eval=5, early_stopping_rounds=30)\n\n    log('at last: best iteration: ', bst.best_iteration, ', best score:', bst.best_score)\n    bst.save_model('lgbmodel.txt', bst.best_iteration)\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5d43841ac885146df65b962e78ee509b9bdea29d"},"cell_type":"code","source":"def predict():\n    log('predict procedure')\n    # 1. load data\n    df_test = get_data('test')\n    gc.collect()\n\n    log('start predicting')\n\n    bst = lgb.Booster(model_file='lgbmodel.txt')  # init model\n    y_pred = bst.predict(data=df_test)\n\n    log('predict done, start writing')\n    pd.DataFrame({'click_id': range(len(y_pred)), 'is_attributed': y_pred}).set_index('click_id').to_csv('./preds.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"77e59146f9da5d77236c6012f9478ab8667e6184"},"cell_type":"code","source":"def plot_importance():\n    ax = plt.subplot(111)\n    plt.interactive(False)\n    bst = lgb.Booster(model_file='lgbmodel.txt')  # init model\n    lgb.plot_importance(bst, ax, )\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"9bf98666a24242602c13997d609a11db0591b451"},"cell_type":"code","source":"TRAIN = True\nHAS_MULTI_VALIDATE = False\nRELEASE = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"d8f9c7b586ab99f95182964038e31ea1ee55071e"},"cell_type":"code","source":"log(os.listdir(\"../input/\"))\n\nif TRAIN:\n    train_without_cv(HAS_MULTI_VALIDATE)\n    plot_importance()\n\nif RELEASE:\n    TRAIN = False\n    predict()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}