{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this notebook¶\n- This is the notebook for the final submission.\n- In order to meet memory and time constraints, we used relatively few features.\n- The details of the features are presented in the following notebooks.  \nhttps://www.kaggle.com/tkyiws/riiid-feature-engineering"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\npd.set_option('display.max_columns', 50)\nimport numpy as np\nimport lightgbm as lgb\nfrom collections import defaultdict\nimport gc\nimport pickle\nimport joblib\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"features = [\n    'content_id',\n    'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    'user_correctness',\n    'content_count',\n    'part',\n    'content_mean',\n    'cumcount_u',\n    'cumcount_p',\n    'attempt',\n    'part_avg',\n    'timestamp_diff1',\n    'timestamp_diff2',\n    'cluster_id',\n    'cumcount_cl',\n    'target_lag',\n    'cluster0_avg',\n    'cluster1_avg',\n    'cluster2_avg',\n    'prior_tag',\n    'task_num',\n    'user_rating',\n    'time_mean_diff',\n]\n\ntarget = 'answered_correctly'\n\nfeatures_dtypes = {\n    'content_id': 'int16',\n    'content_mean': 'float32',\n    'prior_question_elapsed_time': 'float64',\n    'prior_question_had_explanation': 'bool',\n    'user_correctness': 'float32',\n    'content_count': 'int32',\n    'part': 'int8',\n    'cumcount_u': 'uint16',\n    'cumcount_p': 'uint16',\n    'attempt': 'uint16',\n    'part_avg': 'float32',\n    'timestamp_diff1': 'float64',\n    'timestamp_diff2': 'float64',\n    'cluster_id': 'int8',\n    'cumcount_cl': 'uint16',\n    'target_lag': 'int8',\n    'cluster0_avg': 'float32',\n    'cluster1_avg': 'float32',\n    'cluster2_avg': 'float32',\n    'prior_tag': 'int16',\n    'task_num': 'int8',\n    'user_rating': 'float32',\n    'time_mean_diff': 'float32',\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"time_dict1 = joblib.load(\"../input/riiid-inf-data/time_dict1.pkl.zip\")\ntime_dict2 = joblib.load(\"../input/riiid-inf-data/time_dict2.pkl.zip\")\ntime_dict3 = joblib.load(\"../input/riiid-inf-data/time_dict3.pkl.zip\")\n\nquestions_df = pd.read_pickle('../input/riiid-inf-data/questions_df.pickle')\nlectures_df = pd.read_pickle('../input/riiid-inf-data/lectures_df.pickle')\n\npart_null_data = pd.read_pickle('../input/riiid-inf-data/part_null_data.pickle')\ncluster_null_data = pd.read_pickle('../input/riiid-inf-data/cluster_null_data.pickle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_dict_sum = joblib.load(\"../input/riiid-dict-data/user_dict_sum.pkl.zip\")\nuser_dict_count = joblib.load(\"../input/riiid-dict-data/user_dict_count.pkl.zip\")\n\npart_dict_sum = joblib.load(\"../input/riiid-dict-data/part_dict_sum.pkl.zip\")\npart_dict_count = joblib.load(\"../input/riiid-dict-data/part_dict_count.pkl.zip\")\n\ncluster_dict_sum = joblib.load(\"../input/riiid-dict-data/cluster_dict_sum.pkl.zip\")\ncluster_dict_count = joblib.load(\"../input/riiid-dict-data/cluster_dict_count.pkl.zip\")\n\nlag_dict = joblib.load(\"../input/riiid-dict-data/lag_dict.pkl.zip\")\nlast_lecture_dict = joblib.load(\"../input/riiid-dict-data/last_lecture_dict.pkl.zip\")\ncontent_mean_sum_dict = joblib.load(\"../input/riiid-dict-data/content_mean_sum_dict.pkl.zip\")\ntime_adm_dict = joblib.load(\"../input/riiid-dict-data/time_adm_dict.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_state():\n    data = pd.read_pickle('../input/riiid-inf-data/state_data.pickle')   \n    state = dict()\n    \n    for user_id in data['user_id'].unique():\n        state[user_id] = {}\n\n    user_content = data.groupby('user_id')['content_id'].apply(np.array).apply(np.sort).apply(np.unique)\n    user_attempts = data.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8).groupby('user_id').apply(np.array).values\n    user_attempts -= 1\n    \n    del data\n    gc.collect()\n    \n    for user_id, content, attempt in zip(state.keys(), user_content, user_attempts):\n        state[user_id]['user_content_attempts'] = dict(zip(content, attempt))\n        \n    del user_content, user_attempts\n    gc.collect()\n    \n    return state\n\nstate = get_state()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_attempt(test):\n    attempt = []\n    \n    for idx, (user_id, content_id) in enumerate(test[['user_id', 'content_id']].values):\n        if user_id in state:\n            if content_id in state[user_id]['user_content_attempts']:\n                state[user_id]['user_content_attempts'][content_id] = min(6, state[user_id]['user_content_attempts'][content_id] + 1)\n            else:\n                state[user_id]['user_content_attempts'][content_id] = 0\n        else:\n            dict_keys = ['user_content_attempts']\n            dict_default_vals = [dict(zip([content_id],[0]))]\n            state[user_id] = dict(zip(dict_keys, dict_default_vals))\n            \n        attempt.append(state[user_id]['user_content_attempts'][content_id])\n    \n    return attempt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_timestamp_diff(test):\n    timestamp_diff1 = []\n    timestamp_diff2 = []\n    \n    for user_id, timestamp in test[['user_id', 'timestamp']].values:\n        if user_id in time_dict1:     \n            if timestamp > time_dict1[user_id]:  \n                if time_dict2[user_id] is np.nan:\n                    timestamp_diff1.append(timestamp - time_dict1[user_id])\n                    timestamp_diff2.append(np.nan)\n                    time_dict3[user_id] = time_dict2[user_id]\n                    time_dict2[user_id] = time_dict1[user_id]\n                    time_dict1[user_id] = timestamp   \n                else:\n                    timestamp_diff1.append(timestamp - time_dict1[user_id])\n                    timestamp_diff2.append(timestamp - time_dict2[user_id])\n                    time_dict3[user_id] = time_dict2[user_id]\n                    time_dict2[user_id] = time_dict1[user_id]\n                    time_dict1[user_id] = timestamp                           \n            else:\n                if time_dict2[user_id] is np.nan:\n                    timestamp_diff1.append(np.nan)\n                    timestamp_diff2.append(np.nan)               \n                elif time_dict3[user_id] is np.nan:\n                    timestamp_diff1.append(timestamp - time_dict2[user_id])\n                    timestamp_diff2.append(np.nan)                   \n                else:\n                    timestamp_diff1.append(timestamp - time_dict2[user_id])\n                    timestamp_diff2.append(timestamp - time_dict3[user_id])         \n        else:\n            timestamp_diff1.append(np.nan)\n            timestamp_diff2.append(np.nan)\n            time_dict1[user_id] = timestamp\n            time_dict2[user_id] = np.nan\n            time_dict3[user_id] = np.nan\n            \n    return timestamp_diff1, timestamp_diff2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_user_data(test):\n    user_correctness = []\n    cumcount_u = []\n    part_avg = []\n    cumcount_p = []\n    cluster0_avg = []\n    cluster1_avg = []\n    cluster2_avg = []\n    cumcount_cl = []\n    target_lag = []\n    user_rating = []\n    \n    for user_id, part, cluster_id, content_mean in test[['user_id', 'part', 'cluster_id', 'content_mean']].values:\n        try:\n            part_null = part_null_data[part]\n        except:\n            part_null = part_null_data.mean()\n          \n        try:\n            cluster0_null = cluster_null_data[0]\n        except:\n            cluster0_null = cluster_null_data.mean()\n            \n        try:\n            cluster1_null = cluster_null_data[1]\n        except:\n            cluster1_null = cluster_null_data.mean()\n            \n        try:\n            cluster2_null = cluster_null_data[2]\n        except:\n            cluster2_null = cluster_null_data.mean()\n            \n        if user_id in user_dict_sum:\n            user_correctness.append(user_dict_sum[user_id] / user_dict_count[user_id])    \n            cumcount_u.append(min(7500, user_dict_count[user_id]))\n            user_rating.append((user_dict_sum[user_id] - content_mean_sum_dict[user_id]) / user_dict_count[user_id])\n        else:\n            user_correctness.append(0.68)\n            cumcount_u.append(0)\n            user_rating.append(0)\n            \n        k = (user_id, part)\n        if k in part_dict_sum:\n            part_avg.append(part_dict_sum[k] / part_dict_count[k])    \n            cumcount_p.append(min(7500, part_dict_count[k]))\n        else:\n            part_avg.append(part_null)\n            cumcount_p.append(0)\n            \n        k = (user_id, cluster_id)\n        if k in cluster_dict_sum:  \n            cumcount_cl.append(min(7500, cluster_dict_count[k]))\n        else:\n            cumcount_cl.append(0)\n            \n        k = (user_id, 0)\n        if k in cluster_dict_sum:\n            cluster0_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])    \n        else:\n            cluster0_avg.append(cluster0_null)\n            \n        k = (user_id, 1)\n        if k in cluster_dict_sum:\n            cluster1_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])    \n        else:\n            cluster1_avg.append(cluster1_null)\n            \n        k = (user_id, 2)\n        if k in cluster_dict_sum:\n            cluster2_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])    \n        else:\n            cluster2_avg.append(cluster2_null)\n            \n        if user_id in lag_dict:\n            target_lag.append(lag_dict[user_id])\n        else:\n            target_lag.append(1)\n            \n    return user_correctness, cumcount_u, part_avg, cumcount_p, cluster0_avg, cluster1_avg, cluster2_avg, cumcount_cl, target_lag, user_rating","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_prior_tag(test):\n    prior_tag = []\n    \n    for user_id, tag in test[['user_id', 'prior_tag']].values:\n        if tag == -1:\n            if user_id in last_lecture_dict:\n                prior_tag.append(last_lecture_dict[user_id])\n            else:\n                prior_tag.append(-1)\n                last_lecture_dict[user_id] = -1\n        else:\n            prior_tag.append(tag)\n            last_lecture_dict[user_id] = tag\n                \n    return prior_tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_last_lecture_dict(df):\n    df = df.groupby('user_id').tail(1)[['user_id', 'lecture_tag']]\n    df['lecture_tag'].fillna(-1, inplace=True)    \n    \n    for user_id, lecture_tag in df.values:\n        last_lecture_dict[user_id] = lecture_tag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_time_mean_diff(test):  \n    time_mean_diff = []\n    user_list = []\n    time_adm_list = []\n    \n    for user_id, timestamp_diff1 in test[['user_id', 'timestamp_diff1']].values:    \n        time_adm = min(100000, timestamp_diff1)\n        user_list.append(user_id)\n        time_adm_list.append(time_adm)      \n        if user_id in time_adm_dict:\n            time_mean_diff.append(time_adm - time_adm_dict[user_id] / user_dict_count[user_id])\n        else:\n            time_mean_diff.append(0)\n            \n    # dict update\n    for u, t in zip(user_list, time_adm_list):\n        time_adm_dict[u] += t\n                \n    return time_mean_diff","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\nprior_test_df = None\nmodel = lgb.Booster(model_file='../input/riiid-model-42v1/model42v1.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor (test_df, sample_prediction_df) in iter_test:\n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)\n        \n        # dict update\n        for user_id, content_id, part, cluster_id, answered_correctly, content_mean in prior_test_df[['user_id', 'content_id', 'part', 'cluster_id', 'answered_correctly', 'content_mean']].values:          \n            user_dict_sum[user_id] += answered_correctly\n            user_dict_count[user_id] += 1\n            \n            part_dict_sum[(user_id, part)] += answered_correctly\n            part_dict_count[(user_id, part)] += 1\n            \n            cluster_dict_sum[(user_id, cluster_id)] += answered_correctly\n            cluster_dict_count[(user_id, cluster_id)] += 1\n            \n            lag_dict[user_id] = answered_correctly\n            \n            content_mean_sum_dict[user_id] += content_mean\n\n    test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')\n    test_df = pd.merge(test_df, lectures_df, on=['content_id', 'content_type_id'], how='left')\n    prior_test_df = test_df.copy()\n    test_df['prior_tag'] = test_df.groupby('user_id')['lecture_tag'].shift()\n    test_df['prior_tag'].fillna(-1, inplace=True)\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n    \n    # make data\n    test_df['user_correctness'], test_df['cumcount_u'], \\\n    test_df['part_avg'], test_df['cumcount_p'], \\\n    test_df['cluster0_avg'], test_df['cluster1_avg'], test_df['cluster2_avg'], test_df['cumcount_cl'], \\\n    test_df['target_lag'], test_df['user_rating'] = get_user_data(test_df)\n\n    test_df['timestamp_diff1'], test_df['timestamp_diff2'] = get_timestamp_diff(test_df)\n    test_df['attempt'] = get_attempt(test_df)\n    test_df['prior_tag'] = get_prior_tag(test_df)\n    update_last_lecture_dict(prior_test_df)\n    \n    test_df['timestamp_diff1'] = test_df['timestamp_diff1'] / test_df['task_num']\n    test_df['timestamp_diff2'] = test_df['timestamp_diff2'] / test_df['task_num']\n    \n    # missing value\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df['prior_question_elapsed_time'].fillna(22000., inplace=True)\n    test_df['timestamp_diff1'].fillna(25572., inplace=True)\n    test_df['timestamp_diff2'].fillna(53309., inplace=True)\n    \n    test_df['time_mean_diff'] = get_time_mean_diff(test_df)\n    \n    # dtype\n    test_df = test_df.astype(features_dtypes)\n    \n    # predict\n    test_df['answered_correctly'] = model.predict(test_df[features].values)\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[features].head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}