{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Welcome!\n\nThis is the first notebook I ever published, any tips are welcome.\n\nThe trick that boosted my score significantly was to keep track of the attempt of every user for every question, using this discussion as inspiration.\n\nA state was used during the prediction pahse where also the mean user accuracy and answered (correctly) feature were updated.\n\nI will not disclose my features dataset as this notebook is purely meant for inspiration and not for copying and rerunning.\n\nAll steps will be provided of a short explanation, but if you have any questions feel free to ask them in the comments!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.metrics import roc_auc_score\nfrom multiprocessing import cpu_count\nfrom tqdm.notebook import tqdm\n\n# these imports are used to convert the tree to PNG\nfrom cairosvg import svg2png\nfrom PIL import Image\nfrom io import BytesIO\n\nimport gc\nimport os\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VERSION = 'V1E'\nNUM_BOOST_ROUND = 5000\nVERBOSE_EVAL = 10\nMETRICS = ['auc']\nN_ROWS = 99271300\n\ndef get_index_np():\n    return np.arange(N_ROWS) # speeds up dataframe creation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# features are saved as compressed numpy arrays, much more efficient than a pandas DataFrame!\nFEATURES = np.load(f'/kaggle/input/riiid-training-and-prediction-using-a-state-data/train_features_{VERSION}.npz', allow_pickle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"given_features = [\n    'prior_question_elapsed_time',\n]\n\ndeduced_features = [\n    # user features\n    'mean_user_accuracy',\n    'answered_correctly_user',\n    'answered_user',\n    # content features\n    'mean_content_accuracy',\n    # part features\n    'part',\n    # other features\n    'hmean_user_content_accuracy',\n    'attempt',\n]\n\nfeatures = given_features + deduced_features\n\ntarget = 'answered_correctly'\n\n# add categorical features indices\ncategorical_feature = ['part', 'tags', 'tags_label', 'prior_question_had_explanation']\ncategorical_feature_idxs = []\nfor v in categorical_feature:\n    try:\n        categorical_feature_idxs.append(features.index(v))\n    except:\n        pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make train and validation datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_val_idxs(TRAIN_SIZE, VAL_SIZE):\n    train_idxs = []\n    val_idxs = []\n    NEW_USER_FRAC = 1/4 # fraction of new users in \n    np.random.seed(42)\n    \n    # create df with user_ids and indices\n    df = pd.DataFrame(index=get_index_np())\n    for col in ['user_id']:\n        df[col] = FEATURES[col]\n\n    df['index'] = df.index.values.astype(np.uint32)\n    user_id_index = df.groupby('user_id')['index'].apply(np.array)\n    \n    # iterate over users in random order\n    for indices in user_id_index.sample(user_id_index.size, random_state=42):\n        if len(train_idxs) > TRAIN_SIZE:\n            break\n\n        # fill validation data\n        if len(val_idxs) < VAL_SIZE:\n            # add new user\n            if np.random.rand() < NEW_USER_FRAC:\n                val_idxs += list(indices)\n            # randomly split user between train and val otherwise\n            else:\n                offset = np.random.randint(0, indices.size)\n                train_idxs += list(indices[:offset])\n                val_idxs += list(indices[offset:])\n        else:\n            train_idxs += list(indices)\n        \n    return train_idxs, val_idxs\n\ntrain_idxs, val_idxs = get_train_val_idxs(int(50e6), 2.5e6)\nprint(f'len train_idxs: {len(train_idxs)}, len validation_idxs: {len(val_idxs)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_x_y(FEATURES, train_idxs, val_idxs):\n    # create numpy arrays\n    X_train = np.ndarray(shape=(len(train_idxs), len(features)), dtype=np.float32)\n    X_val = np.ndarray(shape=(len(val_idxs), len(features)), dtype=np.float32)\n    \n    # now fill them up\n    for idx, feature in enumerate(tqdm(features)):\n        X_train[:,idx] = FEATURES[feature][train_idxs].astype(np.float32)\n        X_val[:,idx] = FEATURES[feature][val_idxs].astype(np.float32)\n    \n    # add the target\n    y_train = FEATURES[target][train_idxs].astype(np.int8)\n    y_val = FEATURES[target][val_idxs].astype(np.int8)\n                         \n    return X_train, y_train, X_val, y_val\n    \nX_train, y_train, X_val, y_val = make_x_y(FEATURES, train_idxs, val_idxs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'X_train.shape: {X_train.shape}\\t y_train.shape: {y_train.shape}')\nprint(f'X_val.shape: {X_val.shape}\\t y_val.shape: {y_val.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show train features\npd.DataFrame(X_train[:10], columns=features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make train and validation dataset\ntrain_data = lgb.Dataset(\n    data = X_train,\n    label = y_train,\n    categorical_feature = None,\n)\n\nval_data = lgb.Dataset(\n    data = X_val,\n    label = y_val,\n    categorical_feature = None,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train, y_train, X_val, y_val\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NEW from: \nlgbm_params = {\n    'objective': 'binary',\n    'metric': METRICS,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef train():\n    evals_result = {}\n    model = lgb.train(\n        params = lgbm_params,\n        train_set = train_data,\n        valid_sets = [val_data],\n        num_boost_round = NUM_BOOST_ROUND,\n        verbose_eval = VERBOSE_EVAL,\n        evals_result = evals_result,\n        early_stopping_rounds = 10,\n        categorical_feature = categorical_feature_idxs,\n        feature_name = features,\n    )\n\n    # save model\n    model.save_model(f'model_{VERSION}_{NUM_BOOST_ROUND}.lgb')\n    \n    return model, evals_result\n    \nmodel, evals_result = train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training History"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plots the training history\ndef plot_history(evals_result):\n    for metric in METRICS:\n        plt.figure(figsize=(20,8))\n        \n        for key in evals_result.keys():\n            history_len = len(evals_result.get(key)[metric])\n            history = evals_result.get(key)[metric]\n            x_axis = np.arange(1, history_len + 1)\n            plt.plot(x_axis, history, label=key)\n        \n        x_ticks = list(filter(lambda e: (e % (history_len // 100 * 10) == 0) or e == 1, x_axis))\n        plt.xticks(x_ticks, fontsize=12)\n        plt.yticks(fontsize=12)\n\n        plt.title(f'{metric.upper()} History of training', fontsize=18);\n        plt.xlabel('EPOCH', fontsize=16)\n        plt.ylabel(metric.upper(), fontsize=16)\n        \n        if metric in ['auc']:\n            plt.legend(loc='upper left', fontsize=14)\n        else:\n            plt.legend(loc='upper right', fontsize=14)\n        plt.grid()\n        plt.show()\n\nplot_history(evals_result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot the feature importance in terms of gain and split\ndef show_feature_importances(model, importance_type, max_num_features=10**10):\n    feature_importances = pd.DataFrame()\n    feature_importances['feature'] = features\n    feature_importances['value'] = pd.DataFrame(model.feature_importance(importance_type))\n    feature_importances = feature_importances.sort_values(by='value', ascending=False) # sort feature importance\n    feature_importances.to_csv(f'feature_importances_{importance_type}.csv') # write feature importance to csv\n    feature_importances = feature_importances[:max_num_features] # only show max_num_features\n    \n    plt.figure(figsize=(20, 8))\n    plt.xlim([0, feature_importances.value.max()*1.1])\n    plt.title(f'Feature {importance_type}', fontsize=18);\n    sns.barplot(data=feature_importances, x='value', y='feature', palette='rocket');\n    for idx, v in enumerate(feature_importances.value):\n        plt.text(v, idx, \"  {:.2e}\".format(v))\n\nshow_feature_importances(model, 'gain')\nshow_feature_importances(model, 'split')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show tree and save as png\ndef save_tree_diagraph(model):\n    tree_digraph = lgb.create_tree_digraph(model, show_info=['split_gain', 'internal_count'])\n\n    tree_png = svg2png(tree_digraph._repr_svg_(), output_width=3840)\n    tree_png = Image.open(BytesIO(tree_png))\n\n    tree_png.save('create_tree_digraph.png')\n\n    display(tree_png)\n    \nsave_tree_diagraph(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove train and validation data to free memory before prediction phase\ndel train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataframe with question features used for merging with test_df\ndef get_features_questions_df():\n    # create DataFrame of features\n    features_df = pd.DataFrame(index=get_index_np())\n    for col in tqdm(['content_id', 'part', 'tags', 'tags_label', 'mean_content_accuracy']):\n        features_df[col] = FEATURES[col]\n\n    # content features\n    features_questions_df = features_df.groupby('content_id')[[\n        # merge keys\n        'content_id',\n        'part',\n        'tags',\n        'tags_label',\n        # content\n        'mean_content_accuracy',\n    ]].first().reset_index(drop=True).sort_values('content_id')\n    \n    return features_questions_df\n    \nfeatures_questions_df = get_features_questions_df()\nprint(f'features_questions_df, rows: {features_questions_df.shape[0]}')\ndisplay(features_questions_df.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This next function is the most important part of this notebook, it creates a state to keep track of attempt, mean user accuracy and the total amount of (correct) questions answered. This takes ~2 minutes to generate.\n\nThe state is implemented using dictionaries, as this is more efficient than using a DataFrame.\n\nAn example of the state for a user is given below."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_state():\n    # create DataFrame of features\n    features_df = pd.DataFrame(index=get_index_np())\n    for col in tqdm(['user_id', 'content_id', 'answered_correctly']):\n        features_df[col] = FEATURES[col]\n        \n        \n    # compute user features over all train data\n    mean_user_accuracy = features_df.groupby('user_id')['answered_correctly'].mean().values\n    answered_correctly_user = features_df.groupby('user_id')['answered_correctly'].sum().values\n    answered_user = features_df.groupby('user_id')['answered_correctly'].count().values  \n    \n    # fill dictionary with default values\n    state = dict()\n    for user_id in features_df['user_id'].unique():\n        state[user_id] = {}\n    total = len(state.keys())\n        \n    # add user content attempts\n    user_content = features_df.groupby('user_id')['content_id'].apply(np.array).apply(np.sort).apply(np.unique)\n    user_attempts = features_df.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8).groupby('user_id').apply(np.array).values\n    user_attempts -= 1\n    \n    for user_id, content, attempt in tqdm(zip(state.keys(), user_content, user_attempts),total=total):\n        state[user_id]['user_content_attempts'] = dict(zip(content, attempt))\n        \n    del user_content, user_attempts\n    gc.collect()\n    \n    for idx, user_id in enumerate(state.keys()):\n        state[user_id]['mean_user_accuracy'] = mean_user_accuracy[idx]\n        state[user_id]['answered_correctly_user'] = answered_correctly_user[idx]\n        state[user_id]['answered_user'] = answered_user[idx]\n    \n    return state\n\nstate = get_state()\nprint('Example of the state for user 2746, attempt counting starts at 0 as the pandas cumcount function is used to create the attempt feature')\ndisplay(state[2746])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_user_data(state, test_df):\n    # updated data\n    attempt, mean_user_accuracy, answered_correctly_user, answered_user = [], [], [], []\n    \n    for idx, (user_id, content_id) in test_df[['user_id', 'content_id']].iterrows():\n        # check if user exists\n        if user_id in state:\n            # check if user already answered the question, if so update it to a maximum of 4\n            if content_id in state[user_id]['user_content_attempts']:\n                state[user_id]['user_content_attempts'][content_id] = min(4, state[user_id]['user_content_attempts'][content_id] + 1)\n            # if user did not answered the question already, set the number of attempts to 0\n            else:\n                state[user_id]['user_content_attempts'][content_id] = 0\n        \n        # else create user with default values\n        else:\n            dict_keys = ['mean_user_accuracy', 'answered_correctly_user', 'answered_user', 'user_content_attempts']\n            dict_default_vals = [0.680, 0, 0, dict(zip([content_id],[0]))]\n            state[user_id] = dict(zip(dict_keys, dict_default_vals))\n            \n        # add user data to lists\n        attempt.append(state[user_id]['user_content_attempts'][content_id])\n        mean_user_accuracy.append(state[user_id]['mean_user_accuracy'])\n        answered_correctly_user.append(state[user_id]['answered_correctly_user'])\n        answered_user.append(state[user_id]['answered_user'])\n    \n    return attempt, mean_user_accuracy, answered_correctly_user, answered_user","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# updates the user data\ndef update_user_data(state, features_questions_df, prev_test_df):\n    for user_id, content_id, answered_correctly in prev_test_df[['user_id', 'content_id', 'answered_correctly']].values:\n        # update user features\n        state[user_id]['answered_correctly_user'] += answered_correctly\n        state[user_id]['answered_user'] += 1\n        state[user_id]['mean_user_accuracy'] = state[user_id]['answered_correctly_user'] / state[user_id]['answered_user']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make actual prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"During prediction the previous test_df is saved and updated using the prior_group_answers_correct field of the next test_df.\n\nThis data is than used to update the user features in the state."},{"metadata":{"trusted":true},"cell_type":"code","source":"prev_test_df = None\nmean_attempt_acc_factor = FEATURES['mean_attempt_acc_factor']\n\nfor idx, (test_df, _) in tqdm(enumerate(iter_test)):\n    # from 2nd iteration, update user data\n    if prev_test_df is not None:\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        update_user_data(state, features_questions_df, prev_test_df.loc[prev_test_df['content_type_id'] == 0])\n        if idx is 1:\n            display(test_df)\n            display(prev_test_df)\n    \n    # get user data from state and update attempt\n    attempt, mean_user_accuracy, answered_correctly_user, answered_user = get_user_data(state, test_df)\n\n    # set updated user data\n    test_df['attempt'] = attempt\n    test_df['mean_user_accuracy'] = mean_user_accuracy\n    test_df['answered_correctly_user'] = answered_correctly_user\n    test_df['answered_user'] = answered_user\n\n    # merge with all features\n    test_df = features_questions_df.merge(test_df, how='right', on='content_id')\n\n    # fill prior question had explenation\n    test_df['prior_question_elapsed_time'].fillna(23916, inplace=True)\n\n    # add harmonic mean\n    test_df['hmean_user_content_accuracy'] = 2 * (\n        (test_df['mean_user_accuracy'] * test_df['mean_content_accuracy']) /\n        (test_df['mean_user_accuracy'] + test_df['mean_content_accuracy'])\n    )\n\n    test_df['answered_correctly'] = model.predict(test_df[features])\n\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n\n    # set previour test_df\n    prev_test_df = test_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('./submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}