{"cells":[{"metadata":{},"cell_type":"markdown","source":"Based on [this notebook](https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering)\n\nIt seems that iterating over `.itertuples()` is slightly faster than over `.to_numpy()` and `.values`."},{"metadata":{"execution":{"iopub.execute_input":"2020-10-12T18:21:01.107461Z","iopub.status.busy":"2020-10-12T18:21:01.1067Z","iopub.status.idle":"2020-10-12T18:21:02.19541Z","shell.execute_reply":"2020-10-12T18:21:02.194464Z"},"lines_to_next_cell":2,"papermill":{"duration":1.131405,"end_time":"2020-10-12T18:21:02.195556","exception":false,"start_time":"2020-10-12T18:21:01.064151","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import gc\nfrom collections import defaultdict\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## setting\nCV files are generated by [this notebook](https://www.kaggle.com/its7171/cv-strategy)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\nvalid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\ndebug = False\nvalidaten_flg = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Benchmarks\nReference: https://github.com/chiphuyen/just-pandas-things/blob/master/just-pandas-things.ipynb"},{"metadata":{},"cell_type":"markdown","source":"### Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\ndef add_user_feats_values(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        answered_correctly_sum_u_dict[row[0]] += row[1]\n        count_u_dict[row[0]] += 1\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# funcs for user stats with loop\ndef add_user_feats_numpy(df, answered_correctly_sum_u_dict, count_u_dict):\n    acsu = np.zeros(len(df), dtype=np.int32)\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].to_numpy())):\n        acsu[cnt] = answered_correctly_sum_u_dict[row[0]]\n        cu[cnt] = count_u_dict[row[0]]\n        answered_correctly_sum_u_dict[row[0]] += row[1]\n        count_u_dict[row[0]] += 1\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef add_user_feats_tuples(df, answered_correctly_sum_u_dict, count_u_dict):\n    # the sum of questions answered correctly by the user (chronologically)\n    acsu = np.zeros(len(df), dtype=np.int32)\n    # the sum of questions answered by the user\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt, row in enumerate(tqdm(df[['user_id','answered_correctly']].itertuples(), total=df.shape[0])):\n        acsu[cnt] = answered_correctly_sum_u_dict[row.user_id]\n        cu[cnt] = count_u_dict[row.user_id]\n        answered_correctly_sum_u_dict[row.user_id] += row.answered_correctly\n        count_u_dict[row.user_id] += 1\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef add_user_feats_iterrows(df, answered_correctly_sum_u_dict, count_u_dict):\n    # the sum of questions answered correctly by the user (chronologically)\n    acsu = np.zeros(len(df), dtype=np.int32)\n    # the sum of questions answered by the user\n    cu = np.zeros(len(df), dtype=np.int32)\n    for cnt, (_, row) in tqdm(enumerate(df[['user_id','answered_correctly']].iterrows()), total=df.shape[0]):\n        acsu[cnt] = answered_correctly_sum_u_dict[row.user_id]\n        cu[cnt] = count_u_dict[row.user_id]\n        answered_correctly_sum_u_dict[row.user_id] += row.answered_correctly\n        count_u_dict[row.user_id] += 1\n    user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})\n    user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']\n    df = pd.concat([df, user_feats_df], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read data\ndef read_data():\n    field_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[field_needed]\n    train = train.loc[train.content_type_id == False].reset_index(drop=True)\n    return train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### .values"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain = read_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats_values(train, answered_correctly_sum_u_dict, count_u_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly_sum_u.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### .to_numpy()"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = read_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats_numpy(train, answered_correctly_sum_u_dict, count_u_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly_sum_u.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### .itertuples()"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = read_data()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats_tuples(train, answered_correctly_sum_u_dict, count_u_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly_sum_u.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### .iterrows()"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = read_data()\n# Too slow so we have to take a sample (only 5%)\ntrain = train.sample(frac=0.05)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nanswered_correctly_sum_u_dict = defaultdict(int)\ncount_u_dict = defaultdict(int)\ntrain = add_user_feats_iterrows(train, answered_correctly_sum_u_dict, count_u_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.answered_correctly_sum_u.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}