{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%load_ext line_profiler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas\nimport numpy\nimport os\nimport random\n\nrandom.seed(4649893)\nnumpy.random.seed(1192296)\n\npandas.set_option('display.width', 250)\n\ninput_path = '/kaggle/input/riiid-test-answer-prediction/'\ntrain_path = os.path.join(input_path, 'train.csv')\nquestions_path = os.path.join(input_path, 'questions.csv')\nlectures_path = os.path.join(input_path, 'lectures.csv')\n\nworking_path = '/kaggle/working/'\n\ntrain_dtypes = {\n # 'row_id': 'int64',\n 'timestamp': 'int64',\n 'user_id': 'int32',\n 'content_id': 'int16',\n 'content_type_id': 'int8',\n # 'task_container_id': 'int16',\n # 'user_answer': 'int8',\n 'answered_correctly': 'int8',\n 'prior_question_elapsed_time': 'float32',\n # 'prior_question_had_explanation': 'boolean',\n}\n\nquestions_dtypes = {\n 'question_id': 'int16',\n 'bundle_id': 'int16',\n 'correct_answer': 'int8',\n 'part': 'int8',\n 'tags': 'str',\n}\n\nlectures_dtypes = {\n 'lecture_id': 'int16',\n 'part': 'int8',\n 'tag': 'int16',\n 'type_of': 'str',\n}\n\ndef read_train_csv(nrows=None, chunksize=None):\n path = os.path.join('/kaggle/input/riiid-submission-8', 'train_sorted.csv')\n \n if not os.path.exists(path):\n pandas.read_csv(\n train_path,\n dtype=train_dtypes,\n usecols=train_dtypes.keys(),\n ).sort_values(['timestamp', 'user_id']).to_csv(path, index=False)\n \n\n return pandas.read_csv(\n path,\n dtype=train_dtypes,\n usecols=train_dtypes.keys(),\n nrows=nrows,\n chunksize=chunksize,\n )\n\ndef read_questions_csv():\n df = pandas.read_csv(\n questions_path,\n dtype=questions_dtypes,\n usecols=questions_dtypes.keys(),\n index_col='question_id',\n )\n\n df_split_tags = df['tags'].str.split(expand=True).fillna(-1).add_prefix('q_tag_').astype(numpy.int16)\n df = pandas.concat([df.drop('tags', axis=1), df_split_tags], axis=1)\n\n return df\n\ndef read_lectures_csv():\n df = pandas.read_csv(\n lectures_path,\n dtype=lectures_dtypes,\n usecols=lectures_dtypes.keys(),\n index_col='lecture_id',\n )\n\n df['type_of'] = df['type_of'].map({'concept': 0, 'solving question': 1, 'intention': 2, 'starter': 3}).astype(numpy.uint8)\n return df\n\ndef join_df(dfs, how, on):\n if isinstance(dfs[0], pandas.DataFrame):\n result = dfs[0]\n else:\n result = pandas.DataFrame(dfs[0])\n \n for df in dfs[1:]:\n result = result.join(df, how=how, on=on)\n \n return result\n\ndef left_join(a, b, on):\n if not isinstance(b, pandas.DataFrame):\n b = pandas.DataFrame(b)\n \n if isinstance(on, list):\n for i, o in enumerate(on):\n keys = a[o].to_numpy()\n o2 = b.index.names[i]\n b = b.query(f'{o2} in @keys')\n else:\n keys = a[on].to_numpy()\n b = b.query(f'index in @keys')\n \n return a.join(b, on=on)\n\ndef calc_basic_info(df, on, prefix):\n return join_df([\n df.groupby(on)\n .agg({\n 'answered_correctly': ['count'],\n 'prior_question_elapsed_time': ['sum']\n })\n .set_axis(['count', 'time_sum'], axis=1),\n\n df.query('answered_correctly == 1').groupby(on)\n .agg({'prior_question_elapsed_time': ['count', 'sum']})\n .set_axis(['n', 'time_sum'], axis=1)\n .add_suffix('_true'),\n\n df.query('answered_correctly == 0').groupby(on)\n .agg({'prior_question_elapsed_time': ['count', 'sum']})\n .set_axis(['n', 'time_sum'], axis=1)\n .add_suffix('_false'),\n ], how='outer', on=on).add_prefix(f'{prefix}_')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numba\n\n@numba.jitclass([\n ('user_n_answers', numba.uint16[:]),\n ('user_n_corrects', numba.uint16[:]),\n ('user_n_q_parts', numba.uint16[:, :]),\n ('user_n_q_tags', numba.uint16[:, :]),\n])\nclass UserQCount:\n def __init__(self, max_n_users):\n self.user_n_answers = numpy.zeros(max_n_users, dtype=numpy.uint16)\n self.user_n_corrects = numpy.zeros(max_n_users, dtype=numpy.uint16)\n self.user_n_q_parts = numpy.zeros((max_n_users, 7), dtype=numpy.uint16)\n self.user_n_q_tags = numpy.zeros((max_n_users, 188), dtype=numpy.uint16)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, answered_correctly, q_part, q_tag_0, q_tag_1, q_tag_2, q_tag_3, q_tag_4, q_tag_5, train):\n result_user_n_answers = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_user_n_corrects = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_user_n_q_part = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_user_n_q_tags = numpy.zeros((result_n_rows, 6), dtype=numpy.uint16)\n \n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n \n result_user_n_answers[result_index] = self.user_n_answers[user_index]\n result_user_n_corrects[result_index] = self.user_n_corrects[user_index]\n result_user_n_q_part[result_index] = self.user_n_q_parts[user_index, int(q_part[i] - 1)]\n \n if 0 <= q_tag_0[i] < 188:\n result_user_n_q_tags[result_index, 0] = self.user_n_q_tags[user_index, int(q_tag_0[i])]\n if 0 <= q_tag_1[i] < 188:\n result_user_n_q_tags[result_index, 1] = self.user_n_q_tags[user_index, int(q_tag_1[i])]\n if 0 <= q_tag_2[i] < 188:\n result_user_n_q_tags[result_index, 2] = self.user_n_q_tags[user_index, int(q_tag_2[i])]\n if 0 <= q_tag_3[i] < 188:\n result_user_n_q_tags[result_index, 3] = self.user_n_q_tags[user_index, int(q_tag_3[i])]\n if 0 <= q_tag_4[i] < 188:\n result_user_n_q_tags[result_index, 4] = self.user_n_q_tags[user_index, int(q_tag_4[i])]\n if 0 <= q_tag_5[i] < 188:\n result_user_n_q_tags[result_index, 5] = self.user_n_q_tags[user_index, int(q_tag_5[i])]\n \n result_index += 1\n \n if train:\n self.user_n_answers[user_index] += 1\n self.user_n_corrects[user_index] += answered_correctly[i]\n self.user_n_q_parts[user_index, int(q_part[i] - 1)] += 1\n \n if 0 <= q_tag_0[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_0[i])] += 1\n if 0 <= q_tag_1[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_1[i])] += 1\n if 0 <= q_tag_2[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_2[i])] += 1\n if 0 <= q_tag_3[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_3[i])] += 1\n if 0 <= q_tag_4[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_4[i])] += 1\n if 0 <= q_tag_5[i] < 188:\n self.user_n_q_tags[user_index, int(q_tag_5[i])] += 1\n \n return {\n 'user_n_answers': result_user_n_answers,\n 'user_n_corrects': result_user_n_corrects,\n 'user_n_q_part': result_user_n_q_part,\n 'user_n_q_tag_0': result_user_n_q_tags[:, 0].astype(numpy.uint16),\n 'user_n_q_tag_1': result_user_n_q_tags[:, 1].astype(numpy.uint16),\n 'user_n_q_tag_2': result_user_n_q_tags[:, 2].astype(numpy.uint16),\n 'user_n_q_tag_3': result_user_n_q_tags[:, 3].astype(numpy.uint16),\n 'user_n_q_tag_4': result_user_n_q_tags[:, 4].astype(numpy.uint16),\n 'user_n_q_tag_5': result_user_n_q_tags[:, 5].astype(numpy.uint16),\n }\n \n@numba.jitclass([\n ('user_n_lectures', numba.uint16[:]),\n ('user_n_l_parts', numba.uint16[:, :]),\n ('user_n_l_tags', numba.uint16[:, :]),\n ('user_n_l_types', numba.uint16[:, :]),\n])\nclass UserLCount:\n def __init__(self, max_n_users):\n self.user_n_lectures = numpy.zeros(max_n_users, dtype=numpy.uint16)\n self.user_n_l_parts = numpy.zeros((max_n_users, 7), dtype=numpy.uint16)\n self.user_n_l_tags = numpy.zeros((max_n_users, 188), dtype=numpy.uint16)\n self.user_n_l_types = numpy.zeros((max_n_users, 4), dtype=numpy.uint16)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, q_part, l_part, type_of, l_tag, q_tag_0, q_tag_1, q_tag_2, q_tag_3, q_tag_4, q_tag_5, train):\n result_user_n_lectures = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_user_n_l_part = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_user_n_l_tags = numpy.zeros((result_n_rows, 6), dtype=numpy.uint16)\n result_user_n_l_types = numpy.empty((result_n_rows, 4), dtype=numpy.uint16)\n \n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n \n result_user_n_lectures[result_index] = self.user_n_lectures[user_index]\n result_user_n_l_part[result_index] = self.user_n_l_parts[user_index, int(q_part[i] - 1)]\n result_user_n_l_types[result_index, :] = self.user_n_l_types[user_index, :]\n \n if 0 <= q_tag_0[i] < 188:\n result_user_n_l_tags[result_index, 0] = self.user_n_l_tags[user_index, int(q_tag_0[i])]\n if 0 <= q_tag_1[i] < 188:\n result_user_n_l_tags[result_index, 1] = self.user_n_l_tags[user_index, int(q_tag_1[i])]\n if 0 <= q_tag_2[i] < 188:\n result_user_n_l_tags[result_index, 2] = self.user_n_l_tags[user_index, int(q_tag_2[i])]\n if 0 <= q_tag_3[i] < 188:\n result_user_n_l_tags[result_index, 3] = self.user_n_l_tags[user_index, int(q_tag_3[i])]\n if 0 <= q_tag_4[i] < 188:\n result_user_n_l_tags[result_index, 4] = self.user_n_l_tags[user_index, int(q_tag_4[i])]\n if 0 <= q_tag_5[i] < 188:\n result_user_n_l_tags[result_index, 5] = self.user_n_l_tags[user_index, int(q_tag_5[i])]\n \n result_index += 1\n else:\n if train:\n self.user_n_lectures[user_index] += 1\n self.user_n_l_parts[user_index, int(l_part[i] - 1)] += 1\n self.user_n_l_types[user_index, int(type_of[i])] += 1\n self.user_n_l_tags[user_index, int(l_tag[i])] += 1\n \n return {\n 'user_n_lectures': result_user_n_lectures,\n 'user_n_l_part': result_user_n_l_part,\n 'user_n_l_tag_0': result_user_n_l_tags[:, 0].astype(numpy.uint16),\n 'user_n_l_tag_1': result_user_n_l_tags[:, 1].astype(numpy.uint16),\n 'user_n_l_tag_2': result_user_n_l_tags[:, 2].astype(numpy.uint16),\n 'user_n_l_tag_3': result_user_n_l_tags[:, 3].astype(numpy.uint16),\n 'user_n_l_tag_4': result_user_n_l_tags[:, 4].astype(numpy.uint16),\n 'user_n_l_tag_5': result_user_n_l_tags[:, 5].astype(numpy.uint16),\n 'user_n_l_type_0': result_user_n_l_types[:, 0].astype(numpy.uint16),\n 'user_n_l_type_1': result_user_n_l_types[:, 1].astype(numpy.uint16),\n 'user_n_l_type_2': result_user_n_l_types[:, 2].astype(numpy.uint16),\n 'user_n_l_type_3': result_user_n_l_types[:, 3].astype(numpy.uint16),\n }\n \n@numba.jitclass([\n ('user_prev_question_timestamp', numba.int64[:]),\n ('user_prev_lecture_timestamp', numba.int64[:]),\n ('user_prev_question_timestamp_nonzero', numba.int64[:]),\n])\nclass UserInterval:\n def __init__(self, max_n_users):\n self.user_prev_question_timestamp = numpy.zeros(max_n_users, dtype=numpy.int64)\n self.user_prev_lecture_timestamp = numpy.zeros(max_n_users, dtype=numpy.int64)\n self.user_prev_question_timestamp_nonzero = numpy.zeros(max_n_users, dtype=numpy.int64)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, timestamps, train):\n result_interval_last_question = numpy.empty(result_n_rows, dtype=numpy.int64)\n result_interval_last_lecture = numpy.empty(result_n_rows, dtype=numpy.int64)\n result_interval_last_question_nonzero = numpy.empty(result_n_rows, dtype=numpy.int64)\n \n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n result_interval_last_question[result_index] = timestamps[i] - self.user_prev_question_timestamp[user_index]\n result_interval_last_lecture[result_index] = timestamps[i] - self.user_prev_lecture_timestamp[user_index]\n result_interval_last_question_nonzero[result_index] = timestamps[i] - self.user_prev_question_timestamp_nonzero[user_index]\n result_index += 1\n\n if train:\n if is_questions[i]:\n self.user_prev_question_timestamp[user_index] = timestamps[i]\n if self.user_prev_question_timestamp_nonzero[user_index] > timestamps[i]:\n self.user_prev_question_timestamp_nonzero[user_index] = timestamps[i]\n else:\n self.user_prev_lecture_timestamp[user_index] = timestamps[i]\n \n \n return {\n 'user_interval_last_question': result_interval_last_question,\n 'user_interval_last_question_nonzero': result_interval_last_question_nonzero,\n 'user_interval_last_lecture': result_interval_last_lecture,\n }\n \n@numba.jitclass([\n ('rate', numba.float32[:]),\n ('user_moving_accuracy', numba.float32[:, :]),\n])\nclass UserMovingAverage:\n def __init__(self, max_n_users):\n self.rate = numpy.array([1, 10, 100], dtype=numpy.float32)\n self.user_moving_accuracy = numpy.full((max_n_users, len(self.rate)), 0.5, dtype=numpy.float32)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, answered_correctly, train):\n result_user_moving_accuracy = numpy.empty((result_n_rows, len(self.rate)), dtype=numpy.float32)\n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n \n for j, rate in enumerate(self.rate):\n result_user_moving_accuracy[result_index, j] = self.user_moving_accuracy[user_index, j]\n \n result_index += 1\n \n if train:\n for j, rate in enumerate(self.rate):\n self.user_moving_accuracy[user_index, j] *= (1 - (1 / rate))\n self.user_moving_accuracy[user_index, j] += answered_correctly[i] * (1 / rate)\n \n return {\n 'user_accuracy_exponential_mean_1': result_user_moving_accuracy[:, 0].astype(numpy.float32),\n 'user_accuracy_exponential_mean_10': result_user_moving_accuracy[:, 1].astype(numpy.float32),\n 'user_accuracy_exponential_mean_100': result_user_moving_accuracy[:, 2].astype(numpy.float32),\n }\n\n@numba.jitclass([\n ('user_correct_memory', numba.float32[:, :]),\n ('user_timestamp_memory', numba.float32[:, :]),\n])\nclass UserPredictTrend:\n def __init__(self, max_n_users):\n self.user_correct_memory = numpy.full((max_n_users, 100), numpy.nan, dtype=numpy.float32)\n self.user_timestamp_memory = numpy.full((max_n_users, 100), numpy.nan, dtype=numpy.float32)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, answered_correctly, timestamps, train):\n result_user_accuracy_predict = numpy.empty(result_n_rows, dtype=numpy.float32)\n result_user_accuracy_trend = numpy.empty(result_n_rows, dtype=numpy.float32)\n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n \n samples = numpy.count_nonzero(numpy.isfinite(self.user_correct_memory[user_index]))\n \n if samples >= 2:\n x = self.user_timestamp_memory[user_index]\n y = self.user_correct_memory[user_index]\n \n coeff1 = numpy.nansum(x * y)\n coeff2 = numpy.nansum(x)\n coeff3 = numpy.nansum(y)\n coeff4 = numpy.nansum(x ** 2)\n \n a = (coeff1 - (coeff2 * coeff3) / samples) / (coeff4 - (coeff2**2) / samples + 1e-10)\n b = numpy.nansum(y - a * x) / samples\n \n result_user_accuracy_predict[result_index] = timestamps[i] * a + b\n result_user_accuracy_trend[result_index] = a\n else:\n result_user_accuracy_predict[result_index] = 0.65\n result_user_accuracy_trend[result_index] = 0.0\n \n result_index += 1\n \n if train:\n self.user_correct_memory[user_index] = numpy.roll(self.user_correct_memory[user_index], -1)\n self.user_timestamp_memory[user_index] = numpy.roll(self.user_timestamp_memory[user_index], -1)\n self.user_correct_memory[user_index, -1] = answered_correctly[i]\n self.user_timestamp_memory[user_index, -1] = timestamps[i]\n \n return {\n 'user_accuracy_predict': result_user_accuracy_predict,\n 'user_accuracy_trend': result_user_accuracy_trend,\n }\n \n@numba.jitclass([\n ('user_question_history', numba.int16[:, :]),\n])\nclass UserAttempts:\n def __init__(self, max_n_users):\n self.user_question_history = numpy.full((max_n_users, 100), -1, dtype=numpy.int16)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, content_ids, train):\n result_user_question_n_attempts = numpy.empty(result_n_rows, dtype=numpy.uint16)\n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n result_user_question_n_attempts[result_index] = numpy.count_nonzero(self.user_question_history[user_index] == content_ids[i])\n result_index += 1\n \n if train:\n self.user_question_history[user_index] = numpy.roll(self.user_question_history[user_index], 1)\n self.user_question_history[user_index, 0] = content_ids[i]\n \n return {\n 'user_question_n_attempts': result_user_question_n_attempts,\n }\n \n@numba.jitclass([\n ('user_n_answers', numba.uint16[:]),\n ('user_n_corrects', numba.uint16[:]),\n ('user_sum_true_time', numba.float32[:]),\n ('user_sum_false_time', numba.float32[:]),\n])\nclass UserMeanPrevTime:\n def __init__(self, max_n_users):\n self.user_n_answers = numpy.zeros(max_n_users, dtype=numpy.uint16)\n self.user_n_corrects = numpy.zeros(max_n_users, dtype=numpy.uint16)\n self.user_sum_true_time = numpy.zeros(max_n_users, dtype=numpy.float32)\n self.user_sum_false_time = numpy.zeros(max_n_users, dtype=numpy.float32)\n \n def calc(self, result_n_rows, user_indices, user_ids, is_questions, answered_correctly, prior_question_elapsed_time, train):\n result_user_mean_time = numpy.empty(result_n_rows, dtype=numpy.float32)\n result_user_mean_time_true = numpy.empty(result_n_rows, dtype=numpy.float32)\n result_user_mean_time_false = numpy.empty(result_n_rows, dtype=numpy.float32)\n result_index = 0\n \n for i in range(user_ids.shape[0]):\n if is_questions[i]:\n user_index = user_indices[user_ids[i]]\n \n result_user_mean_time[result_index] = (self.user_sum_true_time[user_index] + self.user_sum_false_time[user_index]) / (self.user_n_answers[user_index] + 1e-10)\n result_user_mean_time_true[result_index] = self.user_sum_true_time[user_index] / (self.user_n_answers[user_index] + 1e-10)\n result_user_mean_time_false[result_index] = self.user_sum_false_time[user_index] / (self.user_n_answers[user_index] + 1e-10)\n \n result_index += 1\n \n if train:\n self.user_n_answers[user_index] += 1\n self.user_n_corrects[user_index] += answered_correctly[i]\n if answered_correctly[i]:\n self.user_sum_true_time[user_index] += prior_question_elapsed_time[i]\n else:\n self.user_sum_false_time[user_index] += prior_question_elapsed_time[i]\n \n \n return {\n 'user_mean_time': result_user_mean_time,\n 'user_mean_time_true': result_user_mean_time_true,\n 'user_mean_time_false': result_user_mean_time_false,\n }\n \nclass Statistics:\n def __init__(self):\n self.questions_df = read_questions_csv()\n self.lectures_df = read_lectures_csv()\n self.drop_features = []\n \n self.reset_user()\n \n def append_content(self, nrows=None, chunksize=None):\n df_qs = []\n df_ls = []\n \n for df in read_train_csv(nrows=nrows, chunksize=chunksize):\n df = df.loc[:, ['user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time']]\n \n df_q = df.iloc[df['content_type_id'].to_numpy() == 0]\n df_q = df_q.drop(columns='content_type_id')\n df_q = df_q.join(self.questions_df, on='content_id')\n \n df_qs.append(df_q)\n \n df_l = df.iloc[df['content_type_id'].to_numpy() == 1]\n df_l = df_l.drop(columns=['content_type_id', 'answered_correctly', 'prior_question_elapsed_time'])\n df_l = df_l.join(self.lectures_df, on='content_id')\n \n df_ls.append(df_l)\n \n df_q = pandas.concat(df_qs)\n df_l = pandas.concat(df_ls)\n \n self.bundle_count = self.questions_df['bundle_id'].value_counts().rename('bundle_count')\n \n self.question_info = join_df([\n df_q.groupby('content_id')\n .agg({\n 'answered_correctly': ['mean', 'count'],\n })\n .set_axis(['q_accuracy', 'q_attempt'], axis=1)\n .astype({'q_accuracy': 'float32', 'q_attempt': 'uint32'}),\n ], how='outer', on='content_id')\n\n def reset_user(self):\n max_n_users = 500000\n self.user_q_count = UserQCount(max_n_users)\n self.user_l_count = UserLCount(max_n_users)\n self.user_interval = UserInterval(max_n_users)\n self.user_moving_average = UserMovingAverage(max_n_users)\n self.user_predict_trend = UserPredictTrend(max_n_users)\n self.user_attempts = UserAttempts(max_n_users)\n self.user_mean_time = UserMeanPrevTime(max_n_users)\n self.user_index = numba.typed.Dict.empty(key_type=numba.uint32, value_type=numba.uint32)\n \n @staticmethod\n @numba.njit\n def add_users(user_indices, user_ids):\n for i in range(user_ids.shape[0]):\n if user_ids[i] not in user_indices:\n user_indices[user_ids[i]] = len(user_indices)\n \n def process(self, df, train):\n df = left_join(df, self.questions_df, on='content_id').rename(columns={'part': 'q_part'})\n df = left_join(df, self.lectures_df, on='content_id').rename(columns={'part': 'l_part'})\n df = left_join(df, self.bundle_count, on='bundle_id')\n \n dfs = {column: df[column].to_numpy() for column in df.columns}\n \n statistics.add_users(self.user_index, dfs['user_id'])\n \n is_questions = dfs['content_type_id'] == 0\n \n if 'answered_correctly' not in dfs:\n dfs['answered_correctly'] = numpy.zeros_like(dfs['user_id'])\n \n result_n_rows = numpy.count_nonzero(is_questions)\n result = {}\n result.update(self.user_q_count.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['answered_correctly'], dfs['q_part'], dfs['q_tag_0'], dfs['q_tag_1'], dfs['q_tag_2'], dfs['q_tag_3'], dfs['q_tag_4'], dfs['q_tag_5'], train))\n result.update(self.user_l_count.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['q_part'], dfs['l_part'], dfs['type_of'], dfs['tag'], dfs['q_tag_0'], dfs['q_tag_1'], dfs['q_tag_2'], dfs['q_tag_3'], dfs['q_tag_4'], dfs['q_tag_5'], train))\n result.update(self.user_interval.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['timestamp'], train))\n result.update(self.user_moving_average.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['answered_correctly'], train))\n result.update(self.user_predict_trend.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['answered_correctly'], dfs['timestamp'], train))\n result.update(self.user_attempts.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['content_id'], train))\n result.update(self.user_mean_time.calc(result_n_rows, self.user_index, dfs['user_id'], is_questions, dfs['answered_correctly'], dfs['prior_question_elapsed_time'], train))\n \n result['user_accuracy'] = result['user_n_corrects'] / (result['user_n_answers'] + 1e-10)\n \n df_q = df.iloc[is_questions, :]\n df_q = left_join(df_q, self.question_info, on='content_id')\n result['timestamp_in_hour'] = (df_q['timestamp'].to_numpy() / (60 * 60 * 1000))\n result['question_accuracy'] = df_q['q_accuracy'].to_numpy()\n result['question_n_attempt'] = df_q['q_attempt'].to_numpy()\n result['question_bundle_count'] = df_q['bundle_count'].to_numpy()\n result['part'] = df_q['q_part'].to_numpy() - 1\n result['is_reading_question'] = df_q['q_part'].to_numpy() >= 5\n \n result['q_tag_0'] = df_q['q_tag_0'].to_numpy()\n result['q_tag_1'] = df_q['q_tag_1'].to_numpy()\n result['q_tag_2'] = df_q['q_tag_2'].to_numpy()\n result['q_tag_3'] = df_q['q_tag_3'].to_numpy()\n result['q_tag_4'] = df_q['q_tag_4'].to_numpy()\n result['q_tag_5'] = df_q['q_tag_5'].to_numpy()\n \n result['prior_question_elapsed_time'] = df_q['prior_question_elapsed_time'].to_numpy()\n \n result['accuracy_diff_u_q'] = result['user_accuracy'] - result['question_accuracy']\n result['user_first_attempt'] = result['user_n_answers'] == 0\n \n self.feature_names = [name for name in result.keys() if name not in self.drop_features]\n self.categorical_features = list({'part', 'user_first_attempt', 'q_tag_0', 'q_tag_1', 'q_tag_2', 'q_tag_3', 'q_tag_4', 'q_tag_5'} - set(self.drop_features))\n \n return numpy.stack([result[f] for f in self.feature_names], axis=1).astype(numpy.float32)\n \n def apply_train(self, df):\n return self.process(df, train=True), df.loc[df['content_type_id'].to_numpy() == 0, 'answered_correctly'].to_numpy()\n \n def apply_test(self, df):\n return self.process(df, train=False)\n \ndef sample_dataset(x, t, sample):\n assert(x.shape[0] == t.shape[0])\n total_rows = int(x.shape[0])\n if sample < total_rows:\n index = random.sample(range(total_rows), int(sample))\n return x[index], t[index]\n else:\n return x, t\n \ndef statistics_make_train(statistics, nrows=None, chunksize=None, sample=None):\n xs = []\n ts = []\n \n if sample is not None:\n total_rows = 1e8 if nrows is None else nrows\n sample_rate = sample / total_rows\n \n statistics.reset_user()\n for i, df in enumerate(read_train_csv(nrows=nrows, chunksize=chunksize)):\n print(f'chunk {i}.')\n x, t = statistics.apply_train(df)\n \n if sample is not None:\n x, t = sample_dataset(x, t, int(x.shape[0] * sample_rate * 1.5))\n \n xs.append(x)\n ts.append(t)\n\n x = numpy.concatenate(xs, axis=0)\n t = numpy.concatenate(ts, axis=0)\n \n if sample is not None:\n x, t = sample_dataset(x, t, int(sample))\n \n return x, t\n\nstatistics = Statistics()\nstatistics.append_content(nrows=1e5, chunksize=1e7)\n\n%lprun -f Statistics.process -T lprun_statistics_process.txt statistics_make_train(statistics, nrows=1e5, chunksize=1e6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"statistics.feature_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm\nimport sklearn.model_selection\nimport sklearn.inspection\nimport sklearn.ensemble\nimport random\n\ndef make_validation(statistics, nrows=None, chunksize=None, train_rows=1e6, test_rows=1e6):\n total_rows = train_rows + test_rows\n x, t = statistics_make_train(statistics, nrows=nrows, chunksize=chunksize, sample=total_rows)\n assert(x.shape[0] == t.shape[0])\n total_rows = int(x.shape[0])\n \n index = random.sample(range(total_rows), int(train_rows + test_rows))\n train_index = index[:int(train_rows)]\n test_index = index[int(train_rows):]\n \n return x[train_index], x[test_index], t[train_index], t[test_index]\n\ndef benchmark_classifier(classifier, statistics, nrows=None, chunksize=None, train_rows=1e6, test_rows=1e6):\n train_x, test_x, train_t, test_t = make_validation(statistics, nrows, chunksize, train_rows, test_rows)\n \n print(f'train_x.shape = {train_x.shape} ({train_x.nbytes / (1024**2) : .2f} MB)')\n print(f'train_t.shape = {train_t.shape} ({train_t.nbytes / (1024**2) : .2f} MB)')\n print(f'test_x.shape = {test_x.shape} ({test_x.nbytes / (1024**2) : .2f} MB)')\n print(f'test_t.shape = {test_t.shape} ({test_t.nbytes / (1024**2) : .2f} MB)')\n \n classifier.fit(train_x, train_t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features)\n train_score = sklearn.metrics.roc_auc_score(train_t, classifier.predict_proba(train_x)[:, 1])\n test_score = sklearn.metrics.roc_auc_score(test_t, classifier.predict_proba(test_x)[:, 1])\n \n print(f'Train score : {train_score}')\n print(f'Test score : {test_score}')\n \n importances = calc_feature_importances(classifier, statistics)\n \n result = sklearn.inspection.permutation_importance(classifier, train_x, train_t, scoring='roc_auc')\n permutation_importances = result.importances_mean\n \n # importances['permutation_importance_raw'] = permutation_importances\n importances['permutation_importance'] = 100 * permutation_importances / sum(permutation_importances)\n importances = importances.sort_values('permutation_importance', ascending=False)\n \n print(importances)\n return importances\n\ndef train_classifier(classifier, statistics, nrows=None, chunksize=None):\n x, t = statistics_make_train(statistics, nrows=nrows, chunksize=chunksize)\n \n print(f'train_x.shape = {x.shape} ({x.nbytes / (1024**2) : .2f} MB)')\n print(f'train_t.shape = {t.shape} ({t.nbytes / (1024**2) : .2f} MB)')\n\n classifier.fit(x, t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features)\n\ndef calc_feature_importances(classifier, statistics):\n raw_fi = classifier.feature_importances_\n rate_fi = raw_fi / sum(raw_fi)\n # return pandas.DataFrame(data={'feature_name': statistics.feature_names, 'feature_importance': rate_fi*100, 'feature_importance_raw': raw_fi})\n return pandas.DataFrame(data={'feature_name': statistics.feature_names, 'feature_importance': rate_fi*100})\n\nuse_rows = None\nstatistics.append_content(nrows=use_rows, chunksize=1e7)\n\nclassifier = lightgbm.LGBMClassifier(silent=False, class_weight='balanced', objective='binary')\n\nimportances = benchmark_classifier(classifier, statistics, nrows=use_rows, chunksize=5e6, train_rows=1e6, test_rows=1e6)\nstatistics.drop_features = list(importances.iloc[8:, 0])\n\nbenchmark_classifier(classifier, statistics, nrows=use_rows, chunksize=5e6, train_rows=1e6, test_rows=1e6)\n\nclassifier = sklearn.ensemble.BaggingClassifier(\n base_estimator=lightgbm.LGBMClassifier(silent=False, class_weight='balanced', objective='binary'),\n n_estimators=5,\n max_samples=0.5,\n)\ntrain_classifier(classifier, statistics, nrows=use_rows, chunksize=1e7) \n\n# feature_importances = calc_feature_importances(classifier, statistics).sort_values('feature_importance', ascending=False)\n# print(feature_importances)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm\n# import optuna.integration.lightgbm\n# import sklearn.inspection\n# import sklearn.metrics\n\n# params = {\n# 'objective': 'binary',\n# 'metric': 'auc',\n# 'seed': 375645910,\n# 'is_unbalance': True,\n# }\n\n# def make_validation(statistics, nrows=None, chunksize=None, train_rows=1e6, test_rows=1e6):\n# total_rows = train_rows + test_rows\n# x, t = statistics_make_train(statistics, nrows=nrows, chunksize=chunksize, sample=total_rows)\n# assert(x.shape[0] == t.shape[0])\n# total_rows = int(x.shape[0])\n \n# index = random.sample(range(total_rows), int(train_rows + test_rows))\n# train_index = index[:int(train_rows)]\n# test_index = index[int(train_rows):]\n \n# return x[train_index], x[test_index], t[train_index], t[test_index]\n\n# def feature_selection(statistics, nrows=None, chunksize=None, train_rows=1e6, test_rows=1e6):\n# train_x, test_x, train_t, test_t = make_validation(statistics, nrows, chunksize, train_rows, test_rows)\n \n# print(f'train_x.shape = {train_x.shape} ({train_x.nbytes / (1024**2) : .2f} MB)')\n# print(f'train_t.shape = {train_t.shape} ({train_t.nbytes / (1024**2) : .2f} MB)')\n# print(f'test_x.shape = {test_x.shape} ({test_x.nbytes / (1024**2) : .2f} MB)')\n# print(f'test_t.shape = {test_t.shape} ({test_t.nbytes / (1024**2) : .2f} MB)')\n \n# classifier = lightgbm.LGBMClassifier(silent=False, class_weight='balanced', objective='binary')\n# classifier.fit(train_x, train_t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features)\n \n# train_score = sklearn.metrics.roc_auc_score(train_t, classifier.predict_proba(train_x)[:, 1])\n# test_score = sklearn.metrics.roc_auc_score(test_t, classifier.predict_proba(test_x)[:, 1])\n# print(f'Train score : {train_score}')\n# print(f'Test score : {test_score}')\n \n# feature_importances = classifier.feature_importances_\n# permutation_importances = sklearn.inspection.permutation_importance(classifier, train_x, train_t, scoring='roc_auc').importances_mean\n \n# feature_importances_rate = 100 * feature_importances / sum(feature_importances)\n# permutation_importances_rate = 100 * permutation_importances / sum(permutation_importances)\n \n# importances = pandas.DataFrame(index=statistics.feature_names, data={\n# 'feature_importance': feature_importances_rate,\n# 'permutation_importance': permutation_importances_rate\n# }).sort_values('permutation_importance', ascending=False)\n \n# print(importances)\n \n# statistics.drop_features = importances.index[8:]\n \n# def model_optimize(statistics, nrows=None, chunksize=None, train_rows=1e6, test_rows=1e6):\n# train_x, test_x, train_t, test_t = make_validation(statistics, nrows, chunksize, train_rows, test_rows)\n \n# print(f'train_x.shape = {train_x.shape} ({train_x.nbytes / (1024**2) : .2f} MB)')\n# print(f'train_t.shape = {train_t.shape} ({train_t.nbytes / (1024**2) : .2f} MB)')\n# print(f'test_x.shape = {test_x.shape} ({test_x.nbytes / (1024**2) : .2f} MB)')\n# print(f'test_t.shape = {test_t.shape} ({test_t.nbytes / (1024**2) : .2f} MB)')\n \n# train = lightgbm.Dataset(train_x, train_t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features, free_raw_data=False)\n# test = lightgbm.Dataset(test_x, test_t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features, reference=train, free_raw_data=False)\n \n# classifier = optuna.integration.lightgbm.train(params, train_set=train, valid_sets=test, early_stopping_rounds=10, verbose_eval=1000)\n \n# train_score = sklearn.metrics.roc_auc_score(train_t, classifier.predict(train_x))\n# test_score = sklearn.metrics.roc_auc_score(test_t, classifier.predict(test_x))\n \n# print(f'Train score : {train_score}')\n# print(f'Test score : {test_score}')\n \n# print(f'best_params = {classifier.params}')\n \n# return classifier.params\n\n# def create_classifier(statistics, params, nrows=None, chunksize=None):\n# x, t = statistics_make_train(statistics, nrows=nrows, chunksize=chunksize)\n \n# print(f'train_x.shape = {x.shape} ({x.nbytes / (1024**2) : .2f} MB)')\n# print(f'train_t.shape = {t.shape} ({t.nbytes / (1024**2) : .2f} MB)')\n \n# train = lightgbm.Dataset(x, t, feature_name=statistics.feature_names, categorical_feature=statistics.categorical_features)\n \n# classifier = lightgbm.train(params, train_set=train, verbose_eval=100)\n# return classifier\n\n# use_rows = None\n# statistics.append_content(nrows=use_rows, chunksize=1e7)\n# statistics.drop_features = []\n\n# feature_selection(statistics, nrows=use_rows, chunksize=5e6, train_rows=1e6, test_rows=1e6)\n# best_params = model_optimize(statistics, nrows=use_rows, chunksize=5e6, train_rows=1e6, test_rows=1e6)\n# classifier = create_classifier(statistics, best_params, nrows=use_rows, chunksize=5e6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lightgbm.create_tree_digraph(classifier, show_info=['split_gain', 'internal_count'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\ndef submission(statistics, classifier):\n env = riiideducation.make_env()\n\n prev_test_df = None\n buffer_test_df = None\n counter = 0\n\n iter_test = env.iter_test()\n for (test_df, sample_prediction_df) in iter_test:\n # print(test_df)\n if prev_test_df is not None:\n prior_answered_correctly = eval(test_df['prior_group_answers_correct'].iat[0])\n if prior_answered_correctly:\n prev_test_df['answered_correctly'] = prior_answered_correctly\n prev_test_df = prev_test_df.drop(columns=['prior_group_responses', 'prior_group_answers_correct'])\n\n if buffer_test_df is None:\n buffer_test_df = prev_test_df\n else:\n buffer_test_df = pandas.concat([buffer_test_df, prev_test_df])\n\n if counter % 10 == 0:\n statistics.process(buffer_test_df, train=True)\n buffer_test_df = None\n counter += 1\n\n prev_test_df = test_df.copy()\n\n test_df = test_df.iloc[test_df['content_type_id'].to_numpy() == 0, :]\n\n test_x = statistics.apply_test(test_df)\n prediction = classifier.predict_proba(test_x)\n test_df['answered_correctly'] = prediction[:, 1]\n\n submission = test_df.loc[:, ['row_id', 'answered_correctly']]\n env.predict(submission)\n\n# %lprun -f submission -T lprun_submission.txt submission(statistics, classifier)\nsubmission(statistics, classifier)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}