{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nimport datatable as dt\nimport lightgbm as lgb\nimport gc\nimport psutil\nimport os\nimport sys\nimport math\nimport random\nimport shap\nimport json\nimport riiideducation\n\nfrom collections import defaultdict\nfrom time import time\nfrom datetime import timedelta\nfrom contextlib import contextmanager\nfrom bitarray import bitarray\nfrom typing import List\nfrom pandas import DataFrame, Series","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n######## Global define ########\nDEFAULT_SEED = 42\nPLOT_SHAP = False\n\n# FE\n# df_train['user_id'].nunique() == 393656\nMAX_QUESTIONS = 14000  # 13523 question_id in questions.csv\nVAL_SIZE = 2500000\nUSE_DATA_RATIO = 0.5\n\nTS_SCALING = 1000*3600\n\n# LGBM\nLEARNING_RATE = 0.1  # default = 0.1\nMAX_BIN = 364  # default 255\nNUM_LEAVES = 445  # default = 31\nFEATURE_FRACTION = 0.639\nBAGGING_FRACTION = 0.842\nBAGGING_FREQ = 19\n\nNUM_BOOST_ROUNDS = 10000\nEARLY_STOP_ROUNDS = 20\nVERBOSE_EVAL = 50\n\nTRAIN_FILE_PATH = \"../input/riiid-test-answer-prediction/train.csv\"\nQUESTIONS_FILE_PATH = \"../input/riiid-test-answer-prediction/questions.csv\"\nLECTURES_FILE_PATH = \"../input/riiid-test-answer-prediction/lectures.csv\"\n\n@contextmanager\ndef trace_mem(title):\n    t0 = time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info()[0] / 2. ** 30\n    yield\n    m1 = p.memory_info()[0] / 2. ** 30\n    delta = m1 - m0\n    sign = '+' if delta >= 0 else '-'\n    delta = math.fabs(delta)\n    print(f\"[{m1:.1f}GB({sign}{delta:.2f}GB): {time() - t0:.1f}s] {title} \", file=sys.stderr)\n\ndef seed_everything(seed=DEFAULT_SEED):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\nseed_everything()\n\n# Training features: Update TRA_FEATURES, cat_features, UserFeats, QuesFeats,\n# name_to_extract_fn, get_feat_values when update features!!!\nTRA_FEATURES = [\n    # From row['']:\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    # From QuesFeats:\n    'bundle_id',\n    'part',\n    'tags_encoded',\n    'tags_first1',\n    'tags_last2',\n    'content_count',\n    'content_correctness',\n    # From UserFeats:\n    'user_correctness',\n    'user_correct_cumsum',\n    'residual_user_mean',\n    'pq_elapsed_time_user_mean',\n    'explanation_user_cumsum',\n    'lag_time',\n    'lag_time2',\n    # From global dict:\n    'user_content_attempted',\n    ]\ncat_features = ['bundle_id', 'part', 'tags_encoded', 'tags_first1', 'tags_last2']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n######## Data preparation and FE ########\ntarget_col = 'answered_correctly'\ndata_types_dict = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    # 'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'int8'\n}\n\nwith trace_mem(\"Loading train file, fillna, astype\"):\n    df_train = dt.fread(TRAIN_FILE_PATH, columns=set(data_types_dict.keys())).to_pandas()\n\n    # df_train.isnull().sum().sum() = 2744044 == 2351538 + 392506\n    # df_train['prior_question_elapsed_time'].isnull().sum() = 2351538\n    # df_train['prior_question_had_explanation'].isnull().sum() = 392506\n    df_train['prior_question_had_explanation'].fillna(False, inplace=True)\n    mean_prior_question_elapsed_time = df_train['prior_question_elapsed_time'].mean()\n    df_train['prior_question_elapsed_time'].fillna(mean_prior_question_elapsed_time, inplace=True)\n    assert (0 == df_train.isnull().sum().sum())\n\n    df_train = df_train.astype(data_types_dict)\n    del data_types_dict\n    gc.collect()\n\nwith trace_mem(\"df_train drop lecture rows\"):\n    df_train = df_train[df_train['content_type_id'] == 0].reset_index(drop=True)\n\nwith trace_mem(\"df_train drop useless columns\"):\n    df_train.drop(columns=['content_type_id'], inplace=True)\n\nwith trace_mem(\"Generate feature 'lag_time2'\"):\n    df_train['user_id_copy'] = df_train['user_id']\n    timestamp_groupby_user_task = df_train[['user_id', 'task_container_id', 'row_id', 'user_id_copy', 'timestamp']]. \\\n        groupby(['user_id', 'task_container_id'], sort=False)['row_id', 'user_id_copy', 'timestamp']\n    ts_user_task_agg = timestamp_groupby_user_task.first()\n    del timestamp_groupby_user_task\n    gc.collect()\n    ts_user_task_agg.set_index('row_id', inplace=True)\n    df_train.set_index('row_id', inplace=True)\n\n    # convert timestamp to hours(int64 to float32)\n    ts_user_task_agg['timestamp'] = ts_user_task_agg['timestamp']/TS_SCALING\n    # print(f\"ts_user_task_agg['timestamp'].dtypes: {ts_user_task_agg['timestamp'].dtypes}\")\n    # ts_user_task_agg['timestamp'] = ts_user_task_agg['timestamp'].astype('float32')\n\n    ts_groupby = ts_user_task_agg.groupby('user_id_copy')['timestamp']\n    ts_user_task_agg['timestamp_1'] = ts_groupby.shift(1)\n    ts_user_task_agg['timestamp_1'].fillna(0, inplace=True)\n    ts_user_task_agg['timestamp_1'] = ts_user_task_agg['timestamp_1'].astype('float32')\n    user_max_timestamp_2_agg = ts_user_task_agg.groupby(['user_id_copy'])['timestamp_1'].max()\n    # print(f\"111 user_max_timestamp_2_agg.dtyps: {user_max_timestamp_2_agg.dtypes}\")\n    ts_user_task_agg.drop(columns=['timestamp_1'], inplace=True)\n\n    ts_user_task_agg['timestamp'] = ts_groupby.shift(2)\n    ts_user_task_agg['timestamp'].fillna(0, inplace=True)\n    # print(f\"222 ts_user_task_agg['timestamp'].dtypes: {ts_user_task_agg['timestamp'].dtypes}\")\n    # ts_user_task_agg['timestamp'] = ts_user_task_agg['timestamp'].astype('float32')\n    df_train.drop(columns=['user_id_copy'], inplace=True)\n    del ts_groupby\n    gc.collect()\n\n    df_train['lag_time2'] = 0\n    df_train.loc[ts_user_task_agg.index, 'lag_time2'] = ts_user_task_agg['timestamp']\n    ts_user_task_agg.drop(columns=['user_id_copy', 'timestamp'], inplace=True)\n    del ts_user_task_agg  # How to free ts_user_task_agg mem here???\n    gc.collect()\n\n    df_train['lag_time2'] = df_train['timestamp']/TS_SCALING - df_train['lag_time2']\n    df_train['lag_time2'] = df_train['lag_time2'].astype('float32')\n    # print(f\"555 df_train['lag_time2'].dtypes: {df_train['lag_time2'].dtypes}\")\n\n    lag2_groupby_user_task = df_train[['user_id', 'task_container_id', 'lag_time2']]. \\\n        groupby(['user_id', 'task_container_id'])['lag_time2']\n    df_train['lag_time2'] = lag2_groupby_user_task.transform('first')\n    del lag2_groupby_user_task\n    gc.collect()\n\n    lag2_groupby_user = df_train[['user_id', 'lag_time2']].groupby('user_id')['lag_time2']\n    user_last_lagtime_2_agg = lag2_groupby_user.last()\n    # print(f\"user_last_lagtime_2_agg.dtypes: {user_last_lagtime_2_agg.dtypes}\")\n\n    # df_train.drop(columns=['user_id_copy'], inplace=True)\n    # del timestamp_groupby_user_task, ts_groupby, ts_user_task_agg, lag2_groupby\n    del lag2_groupby_user\n    gc.collect()\n\nwith trace_mem(\"Calculate Train/Val splitting via 'timestamp'\"):\n    user_max_timestamp = df_train[['user_id', 'timestamp']].groupby(['user_id']).agg(['max']).reset_index()\n    user_max_timestamp.columns = ['user_id', 'max_time_stamp']\n    MAX_TIME_STAMP = user_max_timestamp['max_time_stamp'].max()\n\n    def rand_time(max_time_stamp):\n        interval = MAX_TIME_STAMP - max_time_stamp\n        rand_time_stamp = random.randint(0, interval)\n        return rand_time_stamp\n\n    user_max_timestamp['rand_time_stamp'] = user_max_timestamp['max_time_stamp'].apply(rand_time)\n    user_max_timestamp.drop(columns=['max_time_stamp'], inplace=True)\n    df_train = df_train.merge(user_max_timestamp, on='user_id', how='left')\n    del user_max_timestamp\n\n    df_train['virtual_time_stamp'] = df_train['timestamp'] + df_train['rand_time_stamp']\n    ts_arr = (df_train['timestamp']/TS_SCALING).to_numpy()  # .to_numpy(dtype='float32')\n    df_train.drop(columns=['timestamp', 'rand_time_stamp'], inplace=True)\n\n    df_train['val'] = pd.Series([0]*len(df_train), dtype=np.int8)\n\n    use_data_size = int(len(df_train)*USE_DATA_RATIO)\n    print(f\"Use {USE_DATA_RATIO:.0%} of train data: {use_data_size} rows\")\n    df_train.iloc[df_train['virtual_time_stamp'].nlargest(use_data_size).index, -1] = 1\n    df_train.iloc[df_train['virtual_time_stamp'].nlargest(VAL_SIZE).index, -1] = 2\n\n    df_train.drop(columns=['virtual_time_stamp'], inplace=True)\n    gc.collect()\n\n\"\"\"\nwith trace_mem(\"Verify train/val split\"):\n    valid_tmp = df_train[df_train['val'] == 2]\n    train_tmp = df_train[df_train['val'] == 1]\n\n    # check new users and new contents\n    new_user_count = len(valid_tmp[~valid_tmp['user_id'].isin(train_tmp['user_id'])].user_id.unique())\n    new_content_count = len(valid_tmp[~valid_tmp.content_id.isin(train_tmp.content_id)].content_id.unique())\n\n    print(f\"Train rows: {train_tmp.shape[0]}, Valid rows: {valid_tmp.shape[0]}; \"\n          f\"Train target mean: {train_tmp.answered_correctly.mean():.3f}; \"\n          f\"Valid target mean: {valid_tmp.answered_correctly.mean():.3f}; \"\n          f\"new_user_count: {new_user_count}; new_content_count: {new_content_count}\")\n    # Train rows: 47135650, Valid rows: 2500000; Train target mean: 0.657; Valid target mean: 0.645;\n    # new_user_count: 15057(/393656 == 0.038); new_content_count: 3\n\n    # np.random.choice:\n    # Train rows: 46635650, Valid rows: 3000000; Train target mean: 0.658; Valid target mean: 0.646;\n    # new_user_count: 17201; new_content_count: 3\n    del train_tmp, valid_tmp\n    gc.collect()\n\"\"\"\n\nwith trace_mem(\"Generate feature 'lag_time'\"):\n    ## Generate feature 'lag_time': 每个用户的当前'timestamp' - 上一行的'timestamp'\n    df_train['timestamp'] = ts_arr\n    # print(f\"111 df_train['timestamp'].dtypes: {df_train['timestamp'].dtypes}\")\n    del ts_arr\n    gc.collect()\n\n    timestamp_groupby_user = df_train[['user_id', 'timestamp']].groupby('user_id')\n    df_train['lag_time'] = timestamp_groupby_user['timestamp'].shift(1)\n    user_max_timestamp_1_agg = timestamp_groupby_user['timestamp'].max().astype('float32')\n    # print(f\"222 user_max_timestamp_1_agg.dtypes: {user_max_timestamp_1_agg.dtypes}\")\n    del timestamp_groupby_user\n    gc.collect()\n    df_train['lag_time'].fillna(0, inplace=True)  # CV+0.000161 if fillna(0) first\n    # print(f\"333 df_train['lag_time'].dtypes: {df_train['lag_time'].dtypes}\")\n    # df_train['lag_time'] = df_train['lag_time'].astype('float32')\n\n    df_train['lag_time'] = df_train['timestamp'] - df_train['lag_time']\n    df_train['lag_time'] = df_train['lag_time'].astype('float32')  # df_train['lag_time'].max() == 83884261286\n\n    lagtime_groupby = df_train[['user_id', 'task_container_id', 'lag_time']]. \\\n        groupby(['user_id', 'task_container_id'])\n    df_train['lag_time'] = lagtime_groupby['lag_time'].transform('first')\n    # print(f\"444 df_train['lag_time'].dtypes: {df_train['lag_time'].dtypes}\")\n    # df_train.loc[0:1000, ['user_id', 'task_container_id', 'timestamp', 'lag_time', 'lag_time2']].to_csv('./input/lag_time2_after.csv', index=False)\n    df_train.drop(columns=['timestamp'], inplace=True)\n    del lagtime_groupby\n    gc.collect()\n\n    lag_task_groupby_user = df_train[['user_id', 'lag_time', 'task_container_id']].groupby('user_id')\n    user_last_lagtime_1_agg = lag_task_groupby_user['lag_time'].last()\n    # print(f\"555 user_last_lagtime_1_agg.dtypes: {user_last_lagtime_1_agg.dtypes}\")\n    user_last_task_agg = lag_task_groupby_user['task_container_id'].last()\n    df_train.drop(columns=['task_container_id'], inplace=True)\n    del lag_task_groupby_user\n    gc.collect()\n\nwith trace_mem(\"Generate user_content_attempted_dict\"):\n    # Below approach would occupy 1GB RAM\n    # user_content_attempted_dict = df_train.groupby(['user_id'])['content_id'].unique().to_dict(defaultdict(list))\n\n    # [2.7GB(+0.7GB): 40.0s] Generate user_content_attempted_dict\n    user_content_attempted_dict = dict()\n    for _user, _content in zip(df_train['user_id'].to_numpy(), df_train['content_id'].to_numpy()):\n        if _user not in user_content_attempted_dict:\n            a = bitarray(MAX_QUESTIONS, endian='little')\n            a.setall(False)\n            a[_content] = True\n            user_content_attempted_dict[_user] = a\n        else:\n            user_content_attempted_dict[_user][_content] = True\n\n    del a\n    gc.collect()\n\n    def get_user_content_attempted(_user_id, _content_id):\n        if _user_id in user_content_attempted_dict:\n            _attempted = user_content_attempted_dict[_user_id][_content_id]\n            if not _attempted:\n                user_content_attempted_dict[_user_id][_content_id] = True\n\n            return _attempted\n        else:\n            _a = bitarray(MAX_QUESTIONS, endian='little')\n            _a.setall(False)\n            _a[_content_id] = True\n            user_content_attempted_dict[_user_id] = _a\n            return False\n\n## Generate feature 'user_content_attempted': 0 for never attempted, 1 for have attempted before\nwith trace_mem(\"Generate feature 'user_content_attempted'\"):\n    # [5.3GB(+2.11GB): 47.0s] Generate feature 'user_content_attempted'\n    # df_train['user_content_attempted'] = df_train.groupby(['user_id', 'content_id'])['content_id'].\\\n    #     agg(['cumcount']).astype('int8').clip(0, 1)\n\n    # [3.3GB(+0.09GB): 54.7s] Generate feature 'user_content_attempted'\n    df_train[\"user_content_attempted\"] = pd.Series([1]*len(df_train), dtype=np.int8)\n    df_train[\"user_content_attempted\"] = df_train.loc[:, [\"user_id\", \"content_id\", 'user_content_attempted']].\\\n        groupby([\"user_id\", \"content_id\"])[\"user_content_attempted\"].cumsum().clip(0, 2) - 1\n\nwith trace_mem(\"Calculate aggregating values\"):\n    ## Calculate aggregating values\n    groupby_user = df_train[['user_id', 'prior_question_elapsed_time', target_col]].groupby('user_id')\n    groupby_content = df_train[['content_id', target_col, 'prior_question_elapsed_time',\n                                'prior_question_had_explanation']].groupby('content_id')\n\n    user_ques_elapsed_time_sum_agg = groupby_user['prior_question_elapsed_time'].sum()\n    user_target_sum_agg = groupby_user[target_col].sum().astype('int16')  # 每个user的正确数\n    user_count_agg = groupby_user[target_col].count().astype('int16')  # 每个user的个数\n\n    content_target_sum_agg = groupby_content[target_col].sum().astype('int32')  # 每个content的正确数\n    content_count_agg = groupby_content[target_col].count().astype('int32')  # 每个content的个数\n\n    del groupby_user, groupby_content\n    gc.collect()\n\nwith trace_mem(\"Generate feature 'residual_user_mean'\"):\n    df_train.reset_index(drop=True, inplace=True)\n\n    ## Generate feature 'residual' = 当前行的'answered_correctly' - 当前行的content_id的content正确率\n    df_train['residual'] = df_train[target_col] - \\\n                           df_train['content_id'].map(content_target_sum_agg/content_count_agg)\n    df_train['residual'] = df_train['residual']\n\n    ## Generate residual_user_mean as cum_mean\n    residual_groupby_user = df_train[['user_id', 'residual']].groupby('user_id', sort=False)['residual']\n    df_train['lag'] = residual_groupby_user.shift()\n    df_train['lag'].fillna(0, inplace=True)\n\n    lag_groupby_user = df_train[['user_id', 'lag']].groupby('user_id', sort=False)['lag']\n    lag_cumsum, lag_cumcount = lag_groupby_user.cumsum(), lag_groupby_user.cumcount()\n\n    df_train['residual_user_mean'] = lag_cumsum / lag_cumcount\n    df_train['residual_user_mean'].fillna(0, inplace=True)\n    df_train['residual_user_mean'] = df_train['residual_user_mean'].astype('float32')\n\n    user_residual_sum_agg = residual_groupby_user.sum().astype('float32')  # 每个user的'residual'的和\n\n    df_train.drop(columns=['residual', 'lag'], inplace=True)\n    del residual_groupby_user, lag_groupby_user, lag_cumsum, lag_cumcount\n    gc.collect()\n\nwith trace_mem(\"Generate feature 'user_correctness' and 'user_correct_cumsum'\"):\n    ## Generate feature 'user_correctness': current user`s answer accuracy before current row\n    # shift()后第一行'lag'为None, dtype为float64!\n    df_train['lag'] = df_train[['user_id', target_col]].groupby('user_id')[target_col].shift()\n    df_train['lag'].fillna(0, inplace=True)\n    df_train['lag'] = df_train['lag'].astype('int8')\n\n    lag_groupby_user_id = df_train.loc[:, ['user_id', 'lag']].groupby(['user_id'])['lag']\n    lag_cumsum, lag_cumcount = lag_groupby_user_id.cumsum().astype('int16'), \\\n                               lag_groupby_user_id.cumcount().astype('int16')\n\n    df_train['user_correct_cumsum'] = lag_cumsum\n\n    df_train['user_correctness'] = lag_cumsum / lag_cumcount\n    df_train['user_correctness'].fillna(0, inplace=True)\n    df_train['user_correctness'] = df_train['user_correctness'].astype('float32')\n\n    del lag_cumsum, lag_cumcount, lag_groupby_user_id\n    df_train.drop(columns=['lag'], inplace=True)\n    gc.collect()\n\nwith trace_mem(\"Generate feature 'pq_elapsed_time_user_mean'\"):\n    df_train['lag'] = df_train[['user_id', 'prior_question_elapsed_time']]. \\\n        groupby('user_id', sort=False)['prior_question_elapsed_time'].shift()\n    df_train['lag'].fillna(0, inplace=True)\n\n    lag_groupby_user = df_train[['user_id', 'lag']].groupby('user_id', sort=False)['lag']\n    lag_cumsum, lag_cumcount = lag_groupby_user.cumsum(), lag_groupby_user.cumcount()\n\n    df_train['pq_elapsed_time_user_mean'] = lag_cumsum / lag_cumcount\n    df_train['pq_elapsed_time_user_mean'].fillna(0, inplace=True)\n    df_train['pq_elapsed_time_user_mean'] = df_train['pq_elapsed_time_user_mean'].astype('float32')\n\n    df_train.drop(columns=['lag'], inplace=True)\n    del lag_groupby_user, lag_cumsum, lag_cumcount\n    gc.collect()\n\nwith trace_mem(\"Generate feature 'explanation_user_cumsum'\"):\n    explanation_groupby_userid = df_train[['user_id', 'prior_question_had_explanation']]. \\\n        groupby('user_id')['prior_question_had_explanation']\n\n    df_train['lag'] = explanation_groupby_userid.shift()\n    lag_cumsum = df_train[['user_id', 'lag']].groupby('user_id')['lag'].cumsum()\n    df_train['lag'] = lag_cumsum\n\n    df_train.rename(columns={'lag': 'explanation_user_cumsum'}, inplace=True)\n    df_train['explanation_user_cumsum'].fillna(0, inplace=True)\n    df_train['explanation_user_cumsum'] = df_train['explanation_user_cumsum'].astype('int16')\n\n    explanation_sum_agg = explanation_groupby_userid.sum().astype('int16')\n\n    del explanation_groupby_userid, lag_cumsum\n    gc.collect()\n\nwith trace_mem(\"Dropping rows with 'val' == 0\"):\n    df_train = df_train[df_train['val'] != 0].reset_index(drop=True)  # drop rows with 'val' == 0\n\nwith trace_mem(\"Generate feature 'content_count', 'content_correctness'\"):\n    ## Generate feature 'content_count', 'content_correctness'\n    df_train['content_count'] = df_train['content_id'].map(content_count_agg).astype('int32')\n    df_train['content_correctness'] = df_train['content_id'].\\\n        map(content_target_sum_agg/content_count_agg).astype('float32')\n\n## Process questions.csv\nwith trace_mem(\"Process questions.csv\"):\n    NULL_TAG = '255'\n    data_types_dict = {'question_id': 'int16',\n                       'part': 'int8',\n                       'bundle_id': 'int16',\n                       'tags': 'string'}\n    df_questions = pd.read_csv(QUESTIONS_FILE_PATH, usecols=data_types_dict.keys(), dtype=data_types_dict)\n    df_questions['tags'].fillna(NULL_TAG, inplace=True)\n    del data_types_dict\n    assert(0 == df_questions.isnull().sum().sum())\n    df_questions.rename(columns={'question_id': 'content_id'}, inplace=True)\n\n    \"\"\"\n    # Generate feature 'bundle_correctness', CV+0.000167, training time longer\n    df_questions['content_correctness'] = df_questions['content_id'].\\\n        map(content_target_sum_agg/ content_count_agg).astype('float32')\n    bundle_agg = df_questions.groupby('bundle_id')['content_correctness'].agg(['mean'])\n    df_questions['bundle_correctness'] = df_questions['bundle_id'].map(bundle_agg['mean']).astype('float32')\n    del bundle_agg\n    df_questions.drop(columns=['content_correctness'], inplace=True)\n    \"\"\"\n    # Generate feature 'tags_encoded'\n    unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags'].unique())}\n    df_questions['tags_encoded'] = df_questions['tags'].apply(lambda x: unique_tags_combos_keys[x]).astype('int16')\n    del unique_tags_combos_keys\n\n    # Generate feature 'tags_first1' and 'tags_last2'\n    # tag_list.apply(len).value_counts():\n    # (1: 6561)\n    # (2, 171)\n    # (3: 3976)\n    # (4: 2021)\n    # (5: 686)\n    # (6: 108)\n    question_tags_list = df_questions['tags'].apply(lambda x: x.split())  #.apply(lambda x: list(int(t) for t in x))\n\n    df_questions['tags_first1'] = df_questions['tags'].apply(lambda x: x.split()[0])\n    unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags_first1'].unique())}\n    df_questions['tags_first1'] = df_questions['tags_first1'].apply(lambda x: unique_tags_combos_keys[x]).astype('int16')\n    del unique_tags_combos_keys\n\n    df_questions['tags_last2'] = df_questions['tags'].\\\n        apply(lambda x: x.split()[-1] if len(x.split()) == 1 else x.split()[-2]+x.split()[-1])\n    unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags_last2'].unique())}\n    df_questions['tags_last2'] = df_questions['tags_last2'].apply(lambda x: unique_tags_combos_keys[x]).astype('int16')\n\n    # clean up\n    df_questions.drop(columns=['tags'], inplace=True)\n    del unique_tags_combos_keys, question_tags_list\n    gc.collect()\n\n## Merge df_questions to df_train\nwith trace_mem(\"Merge df_questions to df_train\"):\n    ## Merge df_questions to df_train\n    df_train = pd.merge(df_train, df_questions, on='content_id', how='left')\n    assert(0 == df_train.isnull().sum().sum())\n\n## Split to train/val dataset\nwith trace_mem(\"Split to train/val dataset\"):\n    df_val = df_train[df_train['val'] == 2].reset_index(drop=True)\n    df_train = df_train[df_train['val'] != 2].reset_index(drop=True)\n    df_val.drop(columns=['user_id', 'content_id', 'val'], inplace=True)\n    df_train.drop(columns=['user_id', 'content_id', 'val'], inplace=True)\n    print(f\"df_train`s shape {df_train.shape}, df_val`s shape: {df_val.shape}\")\n\n## Save User_feats_dict and Ques_feats_dict for online inference speed up\nwith trace_mem(\"Save User_feats_dict and Ques_feats_dict for online inference\"):\n    def clip(count):\n        return np.clip(count, 1, np.inf)\n\n    # User_feats_dict: {user_id: UserFeats}\n    class UserFeats(object):\n        def __init__(self, user_target_sum=0, user_count=0, explanation_sum=0, prior_question_elapsed_time_sum=0,\n                     user_residual_sum=0, user_max_ts_1=0, user_max_ts_2=0, last_lagtime_1=0,\n                     last_lagtime_2=0, last_task_container=0):\n            self.ans_corr_cnt = user_target_sum\n            self.user_cnt = user_count\n            self.user_acc = user_target_sum/clip(user_count)\n\n            self.user_explanation_sum = explanation_sum\n            self.elapsed_time_sum = prior_question_elapsed_time_sum\n            self.residual_sum = user_residual_sum\n            self.residual_mean = user_residual_sum/clip(user_count)\n            \n            self.user_max_timestamp_1 = user_max_ts_1\n            self.user_max_timestamp_2 = user_max_ts_2\n            self.user_last_lagtime_1 = last_lagtime_1\n            self.user_last_lagtime_2 = last_lagtime_2\n            self.last_task_id = last_task_container\n\n        def update_user_acc(self, _ans_corr):\n            self.ans_corr_cnt += _ans_corr\n            self.user_cnt += 1\n            self.user_acc = self.ans_corr_cnt/clip(self.user_cnt)\n\n        def get_user_acc(self):\n            return self.user_acc\n\n        def get_explanation_cumsum(self, current_explanation):\n            current_explanation_sum = self.user_explanation_sum\n            self.user_explanation_sum += current_explanation\n            return current_explanation_sum\n\n        def get_elapsed_time_mean(self, _elapsed_time):\n            current_elapsed_mean = self.elapsed_time_sum/clip(self.user_cnt)\n            self.elapsed_time_sum += _elapsed_time\n            return current_elapsed_mean\n\n        # suppose update_user_acc() is always called before update_user_residual()\n        # self.user_cnt already been updated, so alway > 0 here\n        def update_user_residual(self, new_residual):\n            ## update self.residual_mean\n            self.residual_sum += new_residual\n            self.residual_mean = self.residual_sum/self.user_cnt  \n\n        def get_residual_mean(self):\n            return self.residual_mean\n\n        def get_lagtime(self, new_timestamp, new_task_id):\n            if new_task_id == self.last_task_id:\n                return self.user_last_lagtime_1, self.user_last_lagtime_2\n\n            self.last_task_id = new_task_id\n            lag_time1, lag_time2 = new_timestamp/TS_SCALING - self.user_max_timestamp_1, \\\n                                   new_timestamp/TS_SCALING - self.user_max_timestamp_2\n            # update lag2 first\n            self.user_max_timestamp_2, self.user_last_lagtime_2 = self.user_max_timestamp_1, lag_time2\n            self.user_max_timestamp_1, self.user_last_lagtime_1 = new_timestamp/TS_SCALING, lag_time1\n\n            return lag_time1, lag_time2\n\n    # Generate User_feats_dict\n    User_feats_dict = defaultdict(UserFeats)\n    for _user_id in user_target_sum_agg.index:\n        _user_feats = UserFeats(user_target_sum=user_target_sum_agg[_user_id], user_count=user_count_agg[_user_id],\n                                explanation_sum=explanation_sum_agg[_user_id],\n                                prior_question_elapsed_time_sum=user_ques_elapsed_time_sum_agg[_user_id],\n                                user_residual_sum=user_residual_sum_agg[_user_id],\n                                user_max_ts_1=user_max_timestamp_1_agg[_user_id],\n                                user_max_ts_2=user_max_timestamp_2_agg[_user_id],\n                                last_lagtime_1=user_last_lagtime_1_agg[_user_id],\n                                last_lagtime_2=user_last_lagtime_2_agg[_user_id],\n                                last_task_container=user_last_task_agg[_user_id])\n        User_feats_dict[_user_id] = _user_feats\n\n    # Ques_feats_dict: {question_id: QuesFeats(ques_feats_list)}\n    class QuesFeats(object):\n        def __init__(self, content_target_sum=0, content_count=0, ques_bundle=1, ques_part=1, ques_tags_encoded=0,\n                     ques_tags_first1=0, ques_tags_last2=0):\n            self.corr_cnt = content_target_sum\n            self.ques_cnt = content_count\n            self.ques_acc = content_target_sum/clip(content_count)\n\n            self.bundle = ques_bundle\n            self.part = ques_part\n            self.tags_encoded = ques_tags_encoded\n            self.tags_first1 = ques_tags_first1\n            self.tags_last2 = ques_tags_last2\n\n        def update_ques_acc(self, _ans_corr):\n            self.corr_cnt += _ans_corr\n            self.ques_cnt += 1\n            self.ques_acc = self.corr_cnt/clip(self.ques_cnt)\n\n        def get_ques_acc(self):\n            return self.ques_acc\n\n    # Generate Ques_feats_dict\n    Ques_feats_dict = defaultdict(QuesFeats)\n    df_ques_tmp = df_questions.set_index('content_id')\n    for _content_id in content_target_sum_agg.index:\n        _ques_feats = QuesFeats(content_target_sum=content_target_sum_agg[_content_id],\n                                content_count=content_count_agg[_content_id],\n                                ques_bundle=df_ques_tmp.loc[_content_id, 'bundle_id'],\n                                ques_part=df_ques_tmp.loc[_content_id, 'part'],\n                                ques_tags_encoded=df_ques_tmp.loc[_content_id, 'tags_encoded'],\n                                ques_tags_first1=df_ques_tmp.loc[_content_id, 'tags_first1'],\n                                ques_tags_last2=df_ques_tmp.loc[_content_id, 'tags_last2'],\n                                )\n        Ques_feats_dict[_content_id] = _ques_feats\n\n    del df_ques_tmp, _user_feats, _ques_feats\n    del user_target_sum_agg, user_count_agg, explanation_sum_agg, content_target_sum_agg, content_count_agg, \\\n        user_residual_sum_agg, user_ques_elapsed_time_sum_agg, user_max_timestamp_1_agg, user_max_timestamp_2_agg, \\\n        user_last_lagtime_1_agg, user_last_lagtime_2_agg, user_last_task_agg\n    gc.collect()\n\nclass FeatureEngineer(object):\n    def __init__(self, user_feats_dict, ques_feats_dict, all_elapsed_time_mean):\n        self._user_feats_dict = user_feats_dict\n        self._ques_feats_dict = ques_feats_dict\n        self._elapsed_time_mean = all_elapsed_time_mean\n        self._init_feat_extract_fns()\n        self._prior_user_ids = []\n        self._prior_content_ids = []\n        self._prior_content_type_ids = []\n\n    def _init_feat_extract_fns(self):\n        name_to_extract_fn = {  # Must match with TRA_FEATURES!\n            ## u_feats: 该user_id所有的feats; q_feats: 该question_id所有的feats;\n            ## row: test_df里当前行; cache: 缓存的通用feats;\n            # From row['']:\n            'prior_question_elapsed_time': lambda u_feats, q_feats, row, cache: self._elapsed_time_mean \\\n                if cache['elapsed_time_isna'] else row['prior_question_elapsed_time'],\n            'prior_question_had_explanation': lambda u_feats, q_feats, row, cache: cache['current_explanation'],\n            # From QuesFeats:\n            'bundle_id': lambda u_feats, q_feats, row, cache: q_feats.bundle,\n            'part': lambda u_feats, q_feats, row, cache: q_feats.part,\n            'tags_encoded': lambda u_feats, q_feats, row, cache: q_feats.tags_encoded,\n            'tags_first1': lambda u_feats, q_feats, row, cache: q_feats.tags_first1,\n            'tags_last2': lambda u_feats, q_feats, row, cache: q_feats.tags_last2,\n            'content_count': lambda u_feats, q_feats, row, cache: q_feats.ques_cnt,\n            'content_correctness': lambda u_feats, q_feats, row, cache: q_feats.get_ques_acc(),\n            # From UserFeats:\n            'user_correctness': lambda u_feats, q_feats, row, cache: u_feats.get_user_acc(),\n            'user_correct_cumsum': lambda u_feats, q_feats, row, cache: u_feats.ans_corr_cnt,\n            'residual_user_mean': lambda u_feats, q_feats, row, cache: u_feats.get_residual_mean(),\n            'pq_elapsed_time_user_mean': lambda u_feats, q_feats, row, cache: \\\n                u_feats.get_elapsed_time_mean(0 if cache['elapsed_time_isna']\n                                              else row['prior_question_elapsed_time']),\n            'explanation_user_cumsum': lambda u_feats, q_feats, row, cache: \\\n                u_feats.get_explanation_cumsum(cache['current_explanation']),\n            'lag_time': lambda u_feats, q_feats, row, cache: cache['lag_times'][0],\n            'lag_time2': lambda u_feats, q_feats, row, cache: cache['lag_times'][1],\n            # From global dict:\n            'user_content_attempted': lambda u_feats, q_feats, row, cache: \\\n                get_user_content_attempted(row['user_id'], row['content_id']),\n        }\n        self._feat_extract_fns = [name_to_extract_fn[name] for name in TRA_FEATURES]\n\n    def _get_row_fvs(self, row: Series, u_feats: UserFeats, q_feats: QuesFeats) -> List[float]:\n        cache = {'current_explanation': 0 if pd.isna(row['prior_question_had_explanation'])\n                                          else row['prior_question_had_explanation'],\n                 'elapsed_time_isna': True if pd.isna(row['prior_question_elapsed_time']) else False,\n                 'lag_times': u_feats.get_lagtime(row['timestamp'], row['task_container_id']),\n                }  # extract stats that is used by multiple features\n        return [fn(u_feats, q_feats, row, cache) for fn in self._feat_extract_fns]\n\n    def _reset_prior_lists(self):\n        self._prior_user_ids, self._prior_content_ids, self._prior_content_type_ids = [], [], []\n\n    def _process_prior_df(self, _prior_targets):\n        len_targets = len(_prior_targets)\n        if len_targets == 0:\n            self._reset_prior_lists()\n            return\n\n        len_prior_user_ids, len_prior_content_ids = len(self._prior_user_ids), len(self._prior_content_ids)\n        if (len_targets != len_prior_user_ids) or (len_prior_user_ids != len_prior_content_ids):\n            print(f\"length not match, len_targets: {len_targets}, len_prior_user_ids: {len_prior_user_ids}, \"\n                  f\"len_prior_content_ids: {len_prior_content_ids}\")\n            self._reset_prior_lists()\n            return\n\n        for _uid, _cid, _target, _content_type in zip(self._prior_user_ids, self._prior_content_ids, _prior_targets,\n                                                      self._prior_content_type_ids):\n            if _content_type == 0:  # only process question rows\n                self._user_feats_dict[_uid].update_user_acc(_target)\n                self._ques_feats_dict[_cid].update_ques_acc(_target)\n                # self._user_feats_dict[_uid].residual_sum += _target - self._ques_feats_dict[_cid].get_ques_acc()\n                self._user_feats_dict[_uid].update_user_residual(_target - self._ques_feats_dict[_cid].get_ques_acc())\n\n        self._reset_prior_lists()\n\n    # Get feature values based on test_df\n    def get_feat_values(self, df: DataFrame):\n        # update prior fields\n        prior_targets_list = json.loads(df['prior_group_answers_correct'].iloc[0])\n        self._process_prior_df(prior_targets_list)\n\n        # generate features\n        fvs, _ques_row_ids = [], []\n\n        for _, row in df.iterrows():\n            self._prior_user_ids.append(row['user_id'])\n            self._prior_content_ids.append(row['content_id'])\n            self._prior_content_type_ids.append(row['content_type_id'])\n\n            if row['content_type_id'] == 0:  # question rows\n                _ques_row_ids.append(row['row_id'])\n                u_feats = self._user_feats_dict[row['user_id']]  # new user will get correct feats(mostly 0)\n                q_feats = self._ques_feats_dict[row['content_id']]\n                fvs.append(self._get_row_fvs(row, u_feats, q_feats))\n\n        return np.array(_ques_row_ids), np.array(fvs)  # np.array(fvs, dtype=np.float32)\n\n######## Define model and train ########\ndef make_x_y_np(df):\n    y = df[target_col].values.astype(np.float32)\n    if not PLOT_SHAP:\n        df.drop(columns=[target_col], inplace=True)\n\n    x = np.ndarray(shape=(df.shape[0], len(TRA_FEATURES)), dtype=np.float32)\n    for idx_, feature_ in enumerate(TRA_FEATURES):\n        x[:, idx_] = df[feature_].values.astype(np.float32)\n        if not PLOT_SHAP:\n            df.drop(columns=[feature_], inplace=True)\n\n    return x, y\n\n\n#df_train.to_csv('./input/df_train.csv', index=False)\n#df_val.to_csv('./input/df_val.csv', index=False)\n\nwith trace_mem(\"Prepare float32 ndarray for LGBM\"):\n    # print(f\"Before make_x_y_np(), df_train = {sys.getsizeof(df_train)}\")\n    X_tra, y_tra = make_x_y_np(df_train)  # Before 4GB, After 0.77GB\n    X_val, y_val = make_x_y_np(df_val)\n    if PLOT_SHAP:\n        del df_train\n    else:\n        del df_train, df_val\n\n    gc.collect()\n\nwith trace_mem(\"Prepare datasets for LGBM\"):  # X_tra+y_tra+X_val+y_val = 5.1GB, lgb_train+lgb_val = 96(copy=False???)\n    lgb_train = lgb.Dataset(X_tra, y_tra,  feature_name=TRA_FEATURES)\n    lgb_val = lgb.Dataset(X_val, y_val, feature_name=TRA_FEATURES)\n\n# After del X_tra, y_tra, X_val, y_val, the RAM overhead will reduce ~5GB during LGBM training,\n# but here: [5.8GB(+0.00GB): 0.0s] del X_tra, y_tra, X_val, y_val ???\nwith trace_mem(\"del X_tra, y_tra, X_val, y_val\"):\n    del X_tra, y_tra, X_val, y_val\n    gc.collect()\n\n## Set hyper parameters and start train\nparams = {'objective': 'binary',\n          'seed': DEFAULT_SEED,\n          'learning_rate': LEARNING_RATE,\n          'metric': 'auc',\n          'max_bin': MAX_BIN,\n          'num_leaves': NUM_LEAVES,  # max number of leaves in one tree, default 31\n          'feature_fraction': FEATURE_FRACTION,  # LightGBM will select FEATURE_FRACTION of features before\n                                                 # training each tree(iteration)\n          'bagging_fraction': BAGGING_FRACTION,\n          'bagging_freq': BAGGING_FREQ,  # Every k-th iteration, LightGBM will randomly select bagging_fraction\n                                         # of the data to use for the next k iterations\n          }\n\ntime_begin = time()\nlgb_model = lgb.train(params,\n                      lgb_train,\n                      valid_sets=[lgb_val],\n                      verbose_eval=VERBOSE_EVAL,\n                      num_boost_round=NUM_BOOST_ROUNDS,\n                      early_stopping_rounds=EARLY_STOP_ROUNDS,\n                      categorical_feature=cat_features)\nprint(f\"Training lgb_model elapsed {str(timedelta(seconds=time()-time_begin))}\")\n\nwith trace_mem(\"del lgb_train, lgb_val\"):\n    del lgb_train, lgb_val\n    gc.collect()\n\n######## Online Inference ########\nfeat_eng = FeatureEngineer(User_feats_dict, Ques_feats_dict, mean_prior_question_elapsed_time)\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()\n\nfor (test_df, _) in iter_test:\n    ques_row_ids, X_test = feat_eng.get_feat_values(test_df)\n    target_preds = lgb_model.predict(X_test)\n    submit_df = pd.DataFrame({'row_id': ques_row_ids, target_col: target_preds})\n    env.predict(submit_df)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}