{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Welcome!\n\nThis is the first notebook I ever published, any tips are welcome.\n\nThe trick that boosted my score significantly was to keep track of the attempt of every user for every question, using this discussion as inspiration.\n\nA state was used during the prediction pahse where also the mean user accuracy and answered (correctly) feature were updated.\n\nI will not disclose my features dataset as this notebook is purely meant for inspiration and not for copying and rerunning.\n\nAll steps will be provided of a short explanation, but if you have any questions feel free to ask them in the comments!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.metrics import roc_auc_score\nfrom multiprocessing import cpu_count\nfrom tqdm.notebook import tqdm\n\n# these imports are used to convert the tree to PNG\nfrom cairosvg import svg2png\nfrom PIL import Image\nfrom io import BytesIO\n\nimport gc\nimport os\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VERSION = 'V1E'\nNUM_BOOST_ROUND = 2500\nVERBOSE_EVAL = 10\nMETRICS = ['auc']\nN_ROWS = 99271300\n\ndef get_index_np():\n    return np.arange(N_ROWS) # speeds up dataframe creation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# features are saved as compressed numpy arrays, much more efficient than a pandas DataFrame!\nFEATURES = np.load(f'/kaggle/input/riiid-answer-correctness-prediction-features/train_features_{VERSION}.npz', allow_pickle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"given_features = [\n    'prior_question_elapsed_time',\n]\n\ndeduced_features = [\n    # user features\n    'mean_user_accuracy',\n    'answered_correctly_user',\n    'answered_user',\n    # content features\n    'mean_content_accuracy',\n    # part features\n    'part',\n    # other features\n    'hmean_user_content_accuracy',\n    'attempt',\n]\n\nfeatures = given_features + deduced_features\n\ntarget = 'answered_correctly'\n\n# add categorical features indices\ncategorical_feature = ['part', 'tags', 'tags_label', 'prior_question_had_explanation']\ncategorical_feature_idxs = []\nfor v in categorical_feature:\n    try:\n        categorical_feature_idxs.append(features.index(v))\n    except:\n        pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make train and validation datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_val_idxs(TRAIN_SIZE, VAL_SIZE):\n    train_idxs = []\n    val_idxs = []\n    NEW_USER_FRAC = 1/4 # fraction of new users in \n    np.random.seed(42)\n    \n    # create df with user_ids and indices\n    df = pd.DataFrame(index=get_index_np())\n    for col in ['user_id']:\n        df[col] = FEATURES[col]\n\n    df['index'] = df.index.values.astype(np.uint32)\n    user_id_index = df.groupby('user_id')['index'].apply(np.array)\n    \n    # iterate over users in random order\n    for indices in user_id_index.sample(user_id_index.size, random_state=42):\n        if len(train_idxs) > TRAIN_SIZE:\n            break\n\n        # fill validation data\n        if len(val_idxs) < VAL_SIZE:\n            # add new user\n            if np.random.rand() < NEW_USER_FRAC:\n                val_idxs += list(indices)\n            # randomly split user between train and val otherwise\n            else:\n                offset = np.random.randint(0, indices.size)\n                train_idxs += list(indices[:offset])\n                val_idxs += list(indices[offset:])\n        else:\n            train_idxs += list(indices)\n        \n    return train_idxs, val_idxs\n\ntrain_idxs, val_idxs = get_train_val_idxs(int(50e6), 2.5e6)\nprint(f'len train_idxs: {len(train_idxs)}, len validation_idxs: {len(val_idxs)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_x_y(FEATURES, train_idxs, val_idxs):\n    # create numpy arrays\n    X_train = np.ndarray(shape=(len(train_idxs), len(features)), dtype=np.float32)\n    X_val = np.ndarray(shape=(len(val_idxs), len(features)), dtype=np.float32)\n    \n    # now fill them up\n    for idx, feature in enumerate(tqdm(features)):\n        X_train[:,idx] = FEATURES[feature][train_idxs].astype(np.float32)\n        X_val[:,idx] = FEATURES[feature][val_idxs].astype(np.float32)\n    \n    # add the target\n    y_train = FEATURES[target][train_idxs].astype(np.int8)\n    y_val = FEATURES[target][val_idxs].astype(np.int8)\n                         \n    return X_train, y_train, X_val, y_val\n    \nX_train, y_train, X_val, y_val = make_x_y(FEATURES, train_idxs, val_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'X_train.shape: {X_train.shape}\\t y_train.shape: {y_train.shape}')\nprint(f'X_val.shape: {X_val.shape}\\t y_val.shape: {y_val.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show train features\npd.DataFrame(X_train[:10], columns=features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make train and validation dataset\ntrain_data = lgb.Dataset(\n    data = X_train,\n    label = y_train,\n    categorical_feature = None,\n)\n\nval_data = lgb.Dataset(\n    data = X_val,\n    label = y_val,\n    categorical_feature = None,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train, y_train, X_val, y_val\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NEW from: \nlgbm_params = {\n    'objective': 'binary',\n    'metric': METRICS,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef train():\n    evals_result = {}\n    model = lgb.train(\n        params = lgbm_params,\n        train_set = train_data,\n        valid_sets = [val_data],\n        num_boost_round = NUM_BOOST_ROUND,\n        verbose_eval = VERBOSE_EVAL,\n        evals_result = evals_result,\n        early_stopping_rounds = 10,\n        categorical_feature = categorical_feature_idxs,\n        feature_name = features,\n    )\n\n    # save model\n    model.save_model(f'model_{VERSION}_{NUM_BOOST_ROUND}.lgb')\n    \n    return model, evals_result\n    \nmodel, evals_result = train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training History"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plots the training history\ndef plot_history(evals_result):\n    for metric in METRICS:\n        plt.figure(figsize=(20,8))\n        \n        for key in evals_result.keys():\n            history_len = len(evals_result.get(key)[metric])\n            history = evals_result.get(key)[metric]\n            x_axis = np.arange(1, history_len + 1)\n            plt.plot(x_axis, history, label=key)\n        \n        x_ticks = list(filter(lambda e: (e % (history_len // 100 * 10) == 0) or e == 1, x_axis))\n        plt.xticks(x_ticks, fontsize=12)\n        plt.yticks(fontsize=12)\n\n        plt.title(f'{metric.upper()} History of training', fontsize=18);\n        plt.xlabel('EPOCH', fontsize=16)\n        plt.ylabel(metric.upper(), fontsize=16)\n        \n        if metric in ['auc']:\n            plt.legend(loc='upper left', fontsize=14)\n        else:\n            plt.legend(loc='upper right', fontsize=14)\n        plt.grid()\n        plt.show()\n\nplot_history(evals_result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot the feature importance in terms of gain and split\ndef show_feature_importances(model, importance_type, max_num_features=10**10):\n    feature_importances = pd.DataFrame()\n    feature_importances['feature'] = features\n    feature_importances['value'] = pd.DataFrame(model.feature_importance(importance_type))\n    feature_importances = feature_importances.sort_values(by='value', ascending=False) # sort feature importance\n    feature_importances.to_csv(f'feature_importances_{importance_type}.csv') # write feature importance to csv\n    feature_importances = feature_importances[:max_num_features] # only show max_num_features\n    \n    plt.figure(figsize=(20, 8))\n    plt.xlim([0, feature_importances.value.max()*1.1])\n    plt.title(f'Feature {importance_type}', fontsize=18);\n    sns.barplot(data=feature_importances, x='value', y='feature', palette='rocket');\n    for idx, v in enumerate(feature_importances.value):\n        plt.text(v, idx, \"  {:.2e}\".format(v))\n\nshow_feature_importances(model, 'gain')\nshow_feature_importances(model, 'split')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show tree and save as png\ndef save_tree_diagraph(model):\n    tree_digraph = lgb.create_tree_digraph(model, show_info=['split_gain', 'internal_count'])\n\n    tree_png = svg2png(tree_digraph._repr_svg_(), output_width=3840)\n    tree_png = Image.open(BytesIO(tree_png))\n\n    tree_png.save('create_tree_digraph.png')\n\n    display(tree_png)\n    \nsave_tree_diagraph(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove train and validation data to free memory before prediction phase\ndel train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataframe with question features used for merging with test_df\ndef get_features_questions_df():\n    # create DataFrame of features\n    features_df = pd.DataFrame(index=get_index_np())\n    for col in tqdm(['content_id', 'part', 'tags', 'tags_label', 'mean_content_accuracy']):\n        features_df[col] = FEATURES[col]\n\n    # content features\n    features_questions_df = features_df.groupby('content_id')[[\n        # merge keys\n        'content_id',\n        'part',\n        'tags',\n        'tags_label',\n        # content\n        'mean_content_accuracy',\n    ]].first().reset_index(drop=True).sort_values('content_id')\n    \n    return features_questions_df\n    \nfeatures_questions_df = get_features_questions_df()\nprint(f'features_questions_df, rows: {features_questions_df.shape[0]}')\ndisplay(features_questions_df.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This next function is the most important part of this notebook, it creates a state to keep track of attempt, mean user accuracy and the total amount of (correct) questions answered. This takes ~2 minutes to generate.\n\nThe state is implemented using dictionaries, as this is more efficient than using a DataFrame.\n\nAn example of the state for a user is given below."},{"metadata":{"trusted":true},"cell_type":"code","source":"#userごとの平均正答数や正答数等の情報が入った連想配列stateを作成する関数（要はuser_table）\ndef get_state():\n    # create DataFrame of features\n    #空のDF作成\n    features_df = pd.DataFrame(index=get_index_np())\n    \n    #必要なカラム追加\n    for col in tqdm(['user_id', 'content_id', 'answered_correctly']):\n        features_df[col] = FEATURES[col]\n        \n        \n    # compute user features over all train data\n    #　userの特徴量（平均正答率、正答数、回答数）をそれぞれ変数に格納\n    mean_user_accuracy = features_df.groupby('user_id')['answered_correctly'].mean().values\n    answered_correctly_user = features_df.groupby('user_id')['answered_correctly'].sum().values\n    answered_user = features_df.groupby('user_id')['answered_correctly'].count().values  \n    \n    \n    # fill dictionary with default values\n    #空のstate連想配列作成\n    state = dict()\n    #user_idごとにさらに入れ子の連想配列作成（JSONチック）\n    for user_id in features_df['user_id'].unique():\n        state[user_id] = {}\n    #ユニークuser数\n    total = len(state.keys())\n        \n    # add user content attempts\n    \n    #user_idごとの回答問題リスト\n    user_content = features_df.groupby('user_id')['content_id'].apply(np.array).apply(np.sort).apply(np.unique)\n    \n    #userごと、問題ごとの回答数\n    user_attempts = features_df.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8).groupby('user_id').apply(np.array).values\n    user_attempts -= 1\n    \n    #stateのuser_idの中にcontent_idがキーで回答数がバリューの辞書を挿入\n    for user_id, content, attempt in tqdm(zip(state.keys(), user_content, user_attempts),total=total):\n        state[user_id]['user_content_attempts'] = dict(zip(content, attempt))\n    #使い終わったuser_content, user_attemptsを消去    \n    del user_content, user_attempts\n    gc.collect()\n    \n    #stateのuser_idの中に平均正答率、正答数、回答数を挿入\n    for idx, user_id in enumerate(state.keys()):\n        state[user_id]['mean_user_accuracy'] = mean_user_accuracy[idx]\n        state[user_id]['answered_correctly_user'] = answered_correctly_user[idx]\n        state[user_id]['answered_user'] = answered_user[idx]\n    \n    return state\n\nstate = get_state()\nprint('Example of the state for user 2746, attempt counting starts at 0 as the pandas cumcount function is used to create the attempt feature')\ndisplay(state[2746])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#stateデータと1ループ分のユーザーリストを渡すと、そのリスト順に該当ユーザーのその問題の回答数、平均正答率、正答数、総回答数をリストで返してくれる関数\ndef get_user_data(state, test_df):\n    # updated data\n    attempt, mean_user_accuracy, answered_correctly_user, answered_user = [], [], [], []\n    \n    #\n    for idx, (user_id, content_id) in test_df[['user_id', 'content_id']].iterrows():\n        # check if user exists\n        #stateのuser_idリストにいる場合（以前の利用履歴があるuserの場合）\n        if user_id in state:\n            # check if user already answered the question, if so update it to a maximum of 4\n            #回答数を反映（上限を4でクリップ）\n            if content_id in state[user_id]['user_content_attempts']:\n                state[user_id]['user_content_attempts'][content_id] = min(4, state[user_id]['user_content_attempts'][content_id] + 1)\n            # if user did not answered the question already, set the number of attempts to 0\n            #回答してなければ0\n            else:\n                state[user_id]['user_content_attempts'][content_id] = 0\n        \n        # else create user with default values\n        #新規ユーザーの場合、デフォルト通りにstateのuser_idの中にデータ挿入\n        else:\n            dict_keys = ['mean_user_accuracy', 'answered_correctly_user', 'answered_user', 'user_content_attempts']\n            dict_default_vals = [0.680, 0, 0, dict(zip([content_id],[0]))]\n            state[user_id] = dict(zip(dict_keys, dict_default_vals))\n            \n        # add user data to lists\n        1ループで渡されたユーザー群のその問題の回答数、平均正答率、正答数、総回答数をリスト化\n        attempt.append(state[user_id]['user_content_attempts'][content_id])\n        mean_user_accuracy.append(state[user_id]['mean_user_accuracy'])\n        answered_correctly_user.append(state[user_id]['answered_correctly_user'])\n        answered_user.append(state[user_id]['answered_user'])\n    #リストデータを返却\n    return attempt, mean_user_accuracy, answered_correctly_user, answered_user","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# updates the user data\n# stateのuser_idの正答数、回答数、平均正答率を更新（features_questions_dfが引数にあるのは謎　試行錯誤の結果？）\ndef update_user_data(state, features_questions_df, prev_test_df):\n    for user_id, content_id, answered_correctly in prev_test_df[['user_id', 'content_id', 'answered_correctly']].values:\n        # update user features\n        state[user_id]['answered_correctly_user'] += answered_correctly\n        state[user_id]['answered_user'] += 1\n        state[user_id]['mean_user_accuracy'] = state[user_id]['answered_correctly_user'] / state[user_id]['answered_user']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make actual prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"During prediction the previous test_df is saved and updated using the prior_group_answers_correct field of the next test_df.\n\nThis data is than used to update the user features in the state."},{"metadata":{"trusted":true},"cell_type":"code","source":"prev_test_df = None\nmean_attempt_acc_factor = FEATURES['mean_attempt_acc_factor']\n\nfor idx, (test_df, _) in tqdm(enumerate(iter_test)):\n    # from 2nd iteration, update user data\n    if prev_test_df is not None:\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        update_user_data(state, c, prev_test_df.loc[prev_test_df['content_type_id'] == 0])\n        if idx is 1:\n            display(test_df)\n            display(prev_test_df)\n    \n    # get user data from state and update attempt\n    attempt, mean_user_accuracy, answered_correctly_user, answered_user = get_user_data(state, test_df)\n\n    # set updated user data\n    test_df['attempt'] = attempt\n    test_df['mean_user_accuracy'] = mean_user_accuracy\n    test_df['answered_correctly_user'] = answered_correctly_user\n    test_df['answered_user'] = answered_user\n\n    # merge with all features\n    test_df = features_questions_df.merge(test_df, how='right', on='content_id')\n\n    # fill prior question had explenation\n    test_df['prior_question_elapsed_time'].fillna(23916, inplace=True)\n\n    # add harmonic mean\n    test_df['hmean_user_content_accuracy'] = 2 * (\n        (test_df['mean_user_accuracy'] * test_df['mean_content_accuracy']) /\n        (test_df['mean_user_accuracy'] + test_df['mean_content_accuracy'])\n    )\n\n    test_df['answered_correctly'] = model.predict(test_df[features])\n\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n\n    # set previour test_df\n    prev_test_df = test_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('./submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}