{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport pickle\nimport psutil\nimport joblib\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'answered_correctly'\n# Features to train and predict\nFEATURES = ['prior_question_elapsed_time', \n            'prior_question_had_explanation', \n            'content_field', \n            'answered_correctly_u_avg', \n            'elapsed_time_u_avg', \n            'explanation_u_avg',\n            'elapsed_time_q_avg', \n            'explanation_q_avg',\n            'explanation_qtrue_avg',\n            'explanation_qfalse_avg',\n            'beta_q', \n            'answered_correctly_uq_count', \n            'timestamp_u_recency_1', \n            'timestamp_u_recency_2', \n            'timestamp_u_recency_3', \n            'timestamp_u_incorrect_recency',\n            'theta_u',\n            'performance_u',\n            'task_container_avg',\n            'tags_avg',\n            'mean_question_accuracy',\n            'std_accuracy',\n            'tags_encoded',\n            'tag_1',\n            'tag_2',\n            'answered_correctly_u_num', \n            'answered_correctly_u_num_field', \n            'answered_correctly_u_avg_field', \n            'answered_correctly_difficulty_weighted_avg', \n            'answered_correctly_difficulty_weighted_avg_field',\n            'min_u_wrong_difficulty',\n            'min_u_wrong_difficulty_field',\n            'max_u_solved_difficulty',\n            'max_u_solved_difficulty_field', \n            'session_u_time', \n            'time_u_to_last_session',\n            'elo_rate',\n            'tags_lsi',\n            'question_elapsed_time_mean',\n            'tag_acc_max',\n            'tag_acc_min',\n            'tag_acc_count',\n            'explanation_qtrue_mean',\n            'explanation_qfalse_mean'\n            ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Functions for theta and beta\ndef get_new_theta(is_good_answer, beta, theta, nb_previous_answers):\n    return theta + learning_rate_theta(nb_previous_answers) * (\n        is_good_answer - probability_of_good_answer(theta, beta)\n    )\n\ndef get_new_beta(is_good_answer, beta, theta, nb_previous_answers):\n    return beta - learning_rate_beta(nb_previous_answers) * (\n        is_good_answer - probability_of_good_answer(theta, beta)\n    )\n\ndef learning_rate_theta(nb_answers):\n    return max(0.3 / (1 + 0.01 * nb_answers), 0.04)\n\ndef learning_rate_beta(nb_answers):\n    return 1 / (1 + 0.05 * nb_answers)\n\ndef probability_of_good_answer(theta, beta, left_asymptote = 1/4):\n    return left_asymptote + (1 - left_asymptote) * sigmoid(theta - beta)\n\ndef sigmoid(x):\n    return 1 / (1 + np.exp(-x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops\ndef add_train_features(df, \n                answered_correctly_u_count, # v1 & v2 \n                answered_correctly_u_sum, # v1 & v2\n                elapsed_time_u_sum, # v1\n                explanation_u_sum, # v1\n                timestamp_u, # v1\n                timestamp_u_incorrect, # v1\n                latest_u_theta, # v1 & v2\n                answered_correctly_q_count, # v1 & v2 \n                answered_correctly_q_sum,  # v1\n                elapsed_time_q_sum, # v1\n                explanation_q_sum, # v1\n                explanation_qtrue_sum, # v1\n                explanation_qtrue_count, # v1\n                latest_q_beta,  # v1 & v2\n                answered_correctly_uq, # v1 & v2\n                question_avg_sum_u, # v1\n                task_container_sum, # v1 & v2\n                task_container_count, # v1 & v2\n                tags_sum, # v1\n                tags_count, # v1\n                answered_correctly_u_count_field, # v2\n                answered_correctly_u_sum_field, # v2\n                answered_correctly_difficulty_weighted_sum, # v2\n                answered_correctly_difficulty_weighted_sum_field, # v2\n                max_solved_difficulty, # v2\n                max_solved_difficulty_field, # v2\n                min_wrong_difficulty, # v2\n                min_wrong_difficulty_field, # v2\n                session_time, # v2\n                since_last_session_time, # v2\n                last_session_start_time, # v2\n                first_action_time, # v2\n                update = True):\n    df['tags'] = df['tags'].\\\n                    apply(lambda ts: [int(x) for x in str(ts).split() if x != 'nan'])\n    # -----------------------------------------------------------------------\n    for _, row in enumerate(df[['user_id', # 0\n                                  'answered_correctly', # 1 \n                                  'content_id', # 2 \n                                  'prior_question_elapsed_time', # 3 \n                                  'prior_question_had_explanation', # 4\n                                  'timestamp', # 5\n                                  'task_container_id', # 6\n                                  'tags', # 7\n                                  'content_field', # 8\n                                  'mean_question_accuracy', # 9\n                                  ]].values):\n        # Client features assignation\n        field = int(row[8]) - 1\n        ###\n        if row[0] not in answered_correctly_u_count_field: \n            answered_correctly_u_count_field[row[0]] = [0] * 7 \n            answered_correctly_u_sum_field[row[0]] = [0] * 7 \n            answered_correctly_difficulty_weighted_sum_field[row[0]] = [0] * 7 \n            max_solved_difficulty_field[row[0]] = [0] * 7 \n            min_wrong_difficulty_field[row[0]] = [0] * 7  \n\n        if first_action_time[row[0]] == 0:\n            first_action_time[row[0]] = row[5]\n            last_session_start_time[row[0]] = row[5]\n        else:\n            if row[5] - last_session_start_time[row[0]] >= 7200 * 1000:\n                since_last_session_time[row[0]] = (row[5] - last_session_start_time[row[0]]) /\\\n                                                    1000 / 3600\n                last_session_start_time[row[0]] = row[5]\n        \n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        answered_correctly_u_count_field[row[0]][field] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n\n        if row[4]:\n            explanation_qtrue_count[row[0]] += 1\n        # ------------------------------------------------------------------\n        # Client Question updates\n        if row[0] not in answered_correctly_uq:\n            answered_correctly_uq[row[0]] = defaultdict(int)\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Other features updates\n        task_container_count[row[6]] += 1\n        \n        tags = row[7]\n        for k in range(len(tags)):\n            tags_count[tags[k]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            theta = latest_u_theta[row[0]]\n            beta = latest_q_beta[row[2]]\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            answered_correctly_u_sum_field[row[0]][field] += row[1] #\n            answered_correctly_difficulty_weighted_sum[row[0]] += row[1] * (1 - row[9]) * 3 #\n            answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] * (1 - row[9]) * 3 #\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                \n                if row[9] > min_wrong_difficulty[row[0]]:\n                    min_wrong_difficulty[row[0]] = row[9] #\n                \n                if row[9] > min_wrong_difficulty_field[row[0]][field]:\n                    min_wrong_difficulty_field[row[0]][field] = row[9] #\n            \n            else:\n                if 1 - row[9] > max_solved_difficulty[row[0]]: \n                    max_solved_difficulty[row[0]] = 1 - row[9] #\n                \n                if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:\n                    max_solved_difficulty_field[row[0]][field] = 1 - row[9] #\n\n            latest_u_theta[row[0]] = get_new_theta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_u_count[row[0]])\n            # ------------------------------------------------------------------\n            # Other features updates\n            task_container_sum[row[6]] += row[1]\n\n            for k in range(len(tags)):\n                tags_sum[tags[k]] += row[1]\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            if row[4]:\n                explanation_qtrue_sum[row[0]] += row[1]\n            latest_q_beta[row[2]] = get_new_beta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_q_count[row[2]])\n            # ------------------------------------------------------------------\n        # Question average sum by user updates\n        question_avg_sum_u[row[0]] += answered_correctly_q_sum[row[2]] /\\\n                                      answered_correctly_q_count[row[2]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops\ndef add_features(df, \n                answered_correctly_u_count, # v1 & v2 \n                answered_correctly_u_sum, # v1 & v2\n                elapsed_time_u_sum, # v1\n                explanation_u_sum, # v1\n                timestamp_u, # v1\n                timestamp_u_incorrect, # v1\n                latest_u_theta, # v1 & v2\n                answered_correctly_q_count, # v1 & v2 \n                answered_correctly_q_sum,  # v1\n                elapsed_time_q_sum, # v1\n                explanation_q_sum, # v1\n                explanation_qtrue_sum, # v1\n                explanation_qtrue_count, # v1\n                latest_q_beta,  # v1 & v2\n                answered_correctly_uq, # v1 & v2\n                question_avg_sum_u, # v1\n                task_container_sum, # v1 & v2\n                task_container_count, # v1 & v2\n                tags_sum, # v1\n                tags_count, # v1\n                answered_correctly_u_count_field, # v2\n                answered_correctly_u_sum_field, # v2\n                answered_correctly_difficulty_weighted_sum, # v2\n                answered_correctly_difficulty_weighted_sum_field, # v2\n                max_solved_difficulty, # v2\n                max_solved_difficulty_field, # v2\n                min_wrong_difficulty, # v2\n                min_wrong_difficulty_field, # v2\n                session_time, # v2\n                since_last_session_time, # v2\n                last_session_start_time, # v2\n                first_action_time, # v2\n                update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32) # v1 & v2\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32) # v1\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32) # v1\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32) # v1\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32) # v1\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32) # v1\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32) # v1\n    theta_u = np.zeros(len(df), dtype = np.float32) # v1 & v2\n    answered_correctly_u_num = np.zeros(len(df), dtype = np.int32) # v2\n    answered_correctly_u_num_field = np.zeros(len(df), dtype = np.int32) # v2\n    answered_correctly_u_avg_field = np.zeros(len(df), dtype = np.float32) # v2\n    answered_correctly_difficulty_weighted_avg = np.zeros(len(df), dtype = np.float32) # v2\n    answered_correctly_difficulty_weighted_avg_field = np.zeros(len(df), dtype = np.float32) # v2\n    max_u_solved_difficulty = np.zeros(len(df), dtype = np.float32) # v2\n    max_u_solved_difficulty_field = np.zeros(len(df), dtype = np.float32) # v2\n    min_u_wrong_difficulty = np.zeros(len(df), dtype = np.float32) # v2\n    min_u_wrong_difficulty_field = np.zeros(len(df), dtype = np.float32) # v2\n    session_u_time = np.zeros(len(df), dtype = np.float32) # v2\n    time_u_to_last_session = np.zeros(len(df), dtype = np.float32) # v2\n    # -----------------------------------------------------------------------\n    # Question features\n#     answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32) # v1\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32) # v1\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32) # v1\n    explanation_qtrue_avg = np.zeros(len(df), dtype = np.float32) # v1\n    explanation_qfalse_avg = np.zeros(len(df), dtype = np.float32) # v1\n    beta_q = np.zeros(len(df), dtype = np.float32) # v1 & v2\n#     answered_correctly_q_num = np.zeros(len(df), dtype = np.int32) # v2\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32) # v1 & v2\n    performance_u = np.zeros(len(df), dtype = np.float32) # v1\n#     hmean_uq = np.zeros(len(df), dtype = np.float32) # v2\n#     hmean_uq_field = np.zeros(len(df), dtype = np.float32) # v2\n    elo_rate = np.zeros(len(df), dtype = np.float32) # v2\n    # -----------------------------------------------------------------------\n    # Other features\n    task_container_avg = np.zeros(len(df), dtype = np.float32) # v1 & v2\n    tags_avg = np.zeros(len(df), dtype = np.float32) # v1\n    # -----------------------------------------------------------------------\n    for num, row in enumerate(df[['user_id', # 0\n                                  'answered_correctly', # 1 \n                                  'content_id', # 2 \n                                  'prior_question_elapsed_time', # 3 \n                                  'prior_question_had_explanation', # 4\n                                  'timestamp', # 5\n                                  'task_container_id', # 6\n                                  'tags', # 7\n                                  'content_field', # 8\n                                  'mean_question_accuracy', # 9\n                                  ]].values):\n        \n        # Client features assignation\n        field = int(row[8]) - 1\n        # ------------------------------------------------------------------\n        answered_correctly_u_num[num] = answered_correctly_u_count[row[0]] #\n\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] /\\\n                                            answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] /\\\n                                      answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] /\\\n                                     answered_correctly_u_count[row[0]]\n            performance_u[num] = answered_correctly_u_avg[num] - \\\n                                 question_avg_sum_u[row[0]] / answered_correctly_u_count[row[0]]\n            theta_u[num] = latest_u_theta[row[0]]\n            answered_correctly_difficulty_weighted_avg[num] = answered_correctly_difficulty_weighted_sum[row[0]] /\\\n                                                answered_correctly_u_count[row[0]] #\n            max_u_solved_difficulty[num] = max_solved_difficulty[row[0]] #\n            min_u_wrong_difficulty[num] = min_wrong_difficulty[row[0]] #\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            performance_u[num] = np.nan\n            theta_u[num] = 0\n            answered_correctly_difficulty_weighted_avg[num] = np.nan #                \n            max_u_solved_difficulty[num] = np.nan #\n            min_u_wrong_difficulty[num] = np.nan #\n\n        ###\n        if row[0] not in answered_correctly_u_count_field: \n            answered_correctly_u_count_field[row[0]] = [0] * 7 \n            answered_correctly_u_sum_field[row[0]] = [0] * 7 \n            answered_correctly_difficulty_weighted_sum_field[row[0]] = [0] * 7 \n            max_solved_difficulty_field[row[0]] = [0] * 7 \n            min_wrong_difficulty_field[row[0]] = [0] * 7 \n        answered_correctly_u_num_field[num] = answered_correctly_u_count_field[row[0]][field] \n\n        if answered_correctly_u_count_field[row[0]][field] != 0:\n            answered_correctly_u_avg_field[num] = answered_correctly_u_sum_field[row[0]][field] /\\\n                                                answered_correctly_u_count_field[row[0]][field]\n            answered_correctly_difficulty_weighted_avg_field[num] = answered_correctly_difficulty_weighted_sum_field[row[0]][field] /\\\n                                                                    answered_correctly_u_count_field[row[0]][field]\n\n            max_u_solved_difficulty_field[num] = max_solved_difficulty_field[row[0]][field]\n            min_u_wrong_difficulty_field[num] = min_wrong_difficulty_field[row[0]][field]\n        else:\n            answered_correctly_u_avg_field[num] = np.nan\n            answered_correctly_difficulty_weighted_avg_field[num] = np.nan\n            \n            max_u_solved_difficulty_field[num] = np.nan\n            min_u_wrong_difficulty_field[num] = np.nan\n\n        if first_action_time[row[0]] == 0:\n            first_action_time[row[0]] = row[5]\n            last_session_start_time[row[0]] = row[5]\n        else:\n            if row[5] - last_session_start_time[row[0]] >= 7200 * 1000:\n                since_last_session_time[row[0]] = (row[5] - last_session_start_time[row[0]]) /\\\n                                                    1000 / 3600\n                last_session_start_time[row[0]] = row[5]\n                session_u_time[num] = 0\n\n            else:\n                session_u_time[num] = (row[5] - last_session_start_time[row[0]]) /\\\n                                        1000 / 60\n        time_u_to_last_session[num] = since_last_session_time[row[0]]\n        ###\n            \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]            \n        # ------------------------------------------------------------------\n        # Question features assignation\n#         answered_correctly_q_num[num] = answered_correctly_q_count[row[2]]\n\n        if answered_correctly_q_count[row[2]] != 0:\n#             answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] /\\\n#                                             answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] /\\\n                                      answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] /\\\n                                     answered_correctly_q_count[row[2]]\n            beta_q[num] = latest_q_beta[row[2]]\n        else:\n#             answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n            beta_q[num] = 0\n\n        if explanation_qtrue_count[row[0]] != 0:\n            explanation_qtrue_avg[num] = explanation_qtrue_sum[row[0]] /\\\n                                        explanation_qtrue_count[row[0]]\n        else:\n            explanation_qtrue_avg[num] = np.nan\n\n        if answered_correctly_u_count[row[0]] - explanation_qtrue_count[row[0]] != 0:\n            explanation_qfalse_avg[num] =  \\\n            (answered_correctly_u_sum[row[0]] - explanation_qtrue_sum[row[0]]) /\\\n            (answered_correctly_u_count[row[0]] - explanation_qtrue_count[row[0]])\n        else:\n            explanation_qfalse_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n#         hmean_uq[num] = 2 * answered_correctly_u_avg[num] * row[9] /\\\n#                         (answered_correctly_u_avg[num] + row[9])\n#         hmean_uq_field[num] = 2 * answered_correctly_u_avg_field[num] * row[9] /\\\n#                             (answered_correctly_u_avg_field[num] + row[9])\n        elo_rate[num] = latest_u_theta[row[0]] - latest_q_beta[row[2]]\n        # ------------------------------------------------------------------\n        # Other features assignation\n        if task_container_count[row[6]] != 0:\n            task_container_avg[num] = task_container_sum[row[6]] /\\\n                                      task_container_count[row[6]]\n        else:\n            task_container_avg[num] = np.nan\n        \n        tags = row[7]\n        tags_means = []\n        for k in range(len(tags)):\n            if tags_count[tags[k]] == 0:\n                continue\n            tags_means.append(tags_sum[tags[k]] / tags_count[tags[k]])\n        if not tags_means:\n            tags_avg[num] = np.nan\n        else:\n            tags_avg[num] = np.mean(tags_means)\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        answered_correctly_u_count_field[row[0]][field] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n\n        if row[4]:\n            explanation_qtrue_count[row[0]] += 1\n        # ------------------------------------------------------------------\n        # Client Question updates\n        if row[0] not in answered_correctly_uq:\n            answered_correctly_uq[row[0]] = defaultdict(int)\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Other features updates\n        task_container_count[row[6]] += 1\n        \n        for k in range(len(tags)):\n            tags_count[tags[k]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            theta = latest_u_theta[row[0]]\n            beta = latest_q_beta[row[2]]\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            answered_correctly_u_sum_field[row[0]][field] += row[1] #\n            answered_correctly_difficulty_weighted_sum[row[0]] += row[1] * (1 - row[9]) * 3 #\n            answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] * (1 - row[9]) * 3 #\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                \n                if row[9] > min_wrong_difficulty[row[0]]:\n                    min_wrong_difficulty[row[0]] = row[9] #\n                \n                if row[9] > min_wrong_difficulty_field[row[0]][field]:\n                    min_wrong_difficulty_field[row[0]][field] = row[9] #\n            \n            else:\n                if 1 - row[9] > max_solved_difficulty[row[0]]: \n                    max_solved_difficulty[row[0]] = 1 - row[9] #\n                \n                if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:\n                    max_solved_difficulty_field[row[0]][field] = 1 - row[9] #\n\n            latest_u_theta[row[0]] = get_new_theta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_u_count[row[0]])\n            # ------------------------------------------------------------------\n            # Other features updates\n            task_container_sum[row[6]] += row[1]\n\n            for k in range(len(tags)):\n                tags_sum[tags[k]] += row[1]\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            if row[4]:\n                explanation_qtrue_sum[row[0]] += row[1]\n            latest_q_beta[row[2]] = get_new_beta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_q_count[row[2]])\n            # ------------------------------------------------------------------\n        # Question average sum by user updates\n        question_avg_sum_u[row[0]] += answered_correctly_q_sum[row[2]] /\\\n                                      answered_correctly_q_count[row[2]]\n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, \n                            'elapsed_time_u_avg': elapsed_time_u_avg, \n                            'explanation_u_avg': explanation_u_avg, \n                            'elapsed_time_q_avg': elapsed_time_q_avg, \n                            'explanation_q_avg': explanation_q_avg,\n                            'explanation_qtrue_avg': explanation_qtrue_avg,\n                            'explanation_qfalse_avg': explanation_qfalse_avg,\n                            'answered_correctly_uq_count': answered_correctly_uq_count, \n                            'timestamp_u_recency_1': timestamp_u_recency_1, \n                            'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, \n                            'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency,\n                            'performance_u': performance_u,\n                            'tags_avg': tags_avg,\n                            'answered_correctly_u_num': answered_correctly_u_num, \n                            'answered_correctly_u_num_field': answered_correctly_u_num_field, \n                            'answered_correctly_u_avg_field': answered_correctly_u_avg_field, \n                            'answered_correctly_difficulty_weighted_avg': answered_correctly_difficulty_weighted_avg, \n                            'answered_correctly_difficulty_weighted_avg_field': answered_correctly_difficulty_weighted_avg_field,\n                            'max_u_solved_difficulty': max_u_solved_difficulty,\n                            'max_u_solved_difficulty_field': max_u_solved_difficulty_field,\n                            'min_u_wrong_difficulty': min_u_wrong_difficulty,\n                            'min_u_wrong_difficulty_field': min_u_wrong_difficulty_field, \n                            'session_u_time': session_u_time, \n                            'time_u_to_last_session': time_u_to_last_session,\n                            'answered_correctly_uq_count': answered_correctly_uq_count,\n                            'theta_u': theta_u,\n                            'beta_q': beta_q,\n                            'elo_rate': elo_rate,\n                            'task_container_avg': task_container_avg,\n                            })\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_features(df, \n                    answered_correctly_u_sum, \n                    answered_correctly_q_sum, \n                    timestamp_u_incorrect,\n                    explanation_qtrue_sum,\n                    task_container_sum,\n                    latest_u_theta,\n                    latest_q_beta,\n                    answered_correctly_u_count,\n                    answered_correctly_q_count,\n                    tags_sum,\n                    answered_correctly_u_sum_field,\n                    answered_correctly_difficulty_weighted_sum,\n                    answered_correctly_difficulty_weighted_sum_field,\n                    max_solved_difficulty,\n                    max_solved_difficulty_field,\n                    min_wrong_difficulty,\n                    min_wrong_difficulty_field,\n                    ):\n    for row in df[['user_id', # 0\n                    'answered_correctly', # 1 \n                    'content_id', # 2\n                    'content_type_id', # 3 \n                    'timestamp', # 4\n                    'prior_question_had_explanation', # 5\n                    'task_container_id', # 6\n                    'tags', # 7\n                    'content_field', # 8\n                    'mean_question_accuracy', # 9\n                    ]].values:\n        if row[3] == 0:\n            field = int(row[8]) - 1\n\n            theta = latest_u_theta[row[0]]\n            beta = latest_q_beta[row[2]]\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            answered_correctly_u_sum_field[row[0]][field] += row[1] #\n            answered_correctly_difficulty_weighted_sum[row[0]] += row[1] * (1 - row[9]) * 3 #\n            answered_correctly_difficulty_weighted_sum_field[row[0]][field] += row[1] * (1 - row[9]) * 3 #\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n\n                \n                if row[9] > min_wrong_difficulty[row[0]]: \n                    min_wrong_difficulty[row[0]] = row[9] #\n                \n                if row[9] > min_wrong_difficulty_field[row[0]][field]:\n                    min_wrong_difficulty_field[row[0]][field] = row[9] #\n            else:\n                if 1 - row[9] > max_solved_difficulty[row[0]]:\n                    max_solved_difficulty[row[0]] = 1 - row[9] #\n                \n                if 1 - row[9] > max_solved_difficulty_field[row[0]][field]:\n                    max_solved_difficulty_field[row[0]][field] = 1 - row[9] #\n\n            latest_u_theta[row[0]] = get_new_theta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_u_count[row[0]])\n\n            # ------------------------------------------------------------------\n            # Other features updates\n            task_container_sum[row[6]] += row[1]\n\n            tags = row[7]\n            for k in range(len(tags)):\n                tags_sum[tags[k]] += row[1]\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            if row[5]:\n                explanation_qtrue_sum[row[0]] += row[1]\n            latest_q_beta[row[2]] = get_new_beta(row[1],\n                                                    beta, theta,\n                                                    answered_correctly_q_count[row[2]])\n            # ------------------------------------------------------------------\n            \n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_and_preprocess(feature_engineering = False, n_split = 3):\n    \n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_metadata_file = '../input/question-metadate-new-new/question_metadata_new_new.csv'\n    question_data_file = '../input/riiid-test-answer-prediction/questions.csv'\n    \n    \n    # Read data\n    feld_needed = ['timestamp', \n                   'user_id', \n                   'answered_correctly', \n                   'content_id', \n                   'content_type_id',\n                   'prior_question_elapsed_time', \n                   'prior_question_had_explanation',\n                   'task_container_id']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    \n            \n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-1000000:]\n        valid = valid.iloc[-1000000:]\n        \n    \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop = True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)\n\n    print('train size: ')\n    print(train.shape)\n\n    print('valid size: ')\n    print(valid.shape)\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_metadata_file)\n    questions_df_ = pd.read_csv(question_data_file)\n    \n#     questions_df_['tags'] = questions_df_['tags'].\\\n#                         apply(lambda ts: [int(x) for x in str(ts).split() if x != 'nan'])\n    questions_df_.rename(columns = {'question_id': 'content_id'}, inplace = True)\n    questions_df = questions_df.merge(questions_df_[['content_id', 'tags']], on = 'content_id', how = 'left')\n    del questions_df_\n    gc.collect()\n    \n    train = train.merge(questions_df[['content_id', 'tags', 'content_field', 'mean_question_accuracy']], on = 'content_id', how = 'left')\n    valid = valid.merge(questions_df[['content_id', 'tags', 'content_field', 'mean_question_accuracy']], on = 'content_id', how = 'left')\n    \n   # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    latest_u_theta = defaultdict(np.float32)\n\n    answered_correctly_u_count_field = {} #\n    answered_correctly_u_sum_field = {} #\n    answered_correctly_difficulty_weighted_sum = defaultdict(np.float32) #\n    answered_correctly_difficulty_weighted_sum_field = {} #\n    max_solved_difficulty = defaultdict(np.float32) #\n    max_solved_difficulty_field = {} #\n    min_wrong_difficulty = defaultdict(np.float32) #\n    min_wrong_difficulty_field = {} #\n    session_time = defaultdict(np.float32) #\n    since_last_session_time = defaultdict(np.float32) #\n    last_session_start_time = defaultdict(np.float32) #\n    first_action_time = defaultdict(np.float32) #\n  \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    explanation_qtrue_sum = defaultdict(int)\n    explanation_qtrue_count = defaultdict(int)\n    latest_q_beta = defaultdict(np.float32)\n    \n    # Client Question dictionary\n    answered_correctly_uq = {}\n    question_avg_sum_u = defaultdict(np.float32)\n\n    # Other features dictionary\n    task_container_sum = defaultdict(int)\n    task_container_count = defaultdict(int)\n    tags_sum = defaultdict(int)\n    tags_count = defaultdict(int)\n\n    # split the train set into n_split subsets\n    l = int(len(train) // n_split)\n    \n    for n in range(n_split - 1):        \n        train.iloc[int(l * n) : int(l * (n + 1))].to_csv('train_%s.csv' % str(n))\n    \n    train.iloc[int(l * (n_split - 1)):].to_csv('train_%s.csv' % str(n+1))\n    \n    del train\n    gc.collect()\n    \n    print('User feature calculation started...')\n    print('\\n')\n    for n in range(n_split):\n        train = pd.read_csv('train_%s.csv' % str(n))\n        print(train.shape)\n        add_train_features(train, \n                            answered_correctly_u_count, # v1 & v2 \n                            answered_correctly_u_sum, # v1 & v2\n                            elapsed_time_u_sum, # v1\n                            explanation_u_sum, # v1\n                            timestamp_u, # v1\n                            timestamp_u_incorrect, # v1\n                            latest_u_theta, # v1 & v2\n                            answered_correctly_q_count, # v1 & v2 \n                            answered_correctly_q_sum,  # v1\n                            elapsed_time_q_sum, # v1\n                            explanation_q_sum, # v1\n                            explanation_qtrue_sum, # v1\n                            explanation_qtrue_count, # v1\n                            latest_q_beta,  # v1 & v2\n                            answered_correctly_uq, # v1 & v2\n                            question_avg_sum_u, # v1\n                            task_container_sum, # v1 & v2\n                            task_container_count, # v1 & v2\n                            tags_sum, # v1\n                            tags_count, # v1\n                            answered_correctly_u_count_field, # v2\n                            answered_correctly_u_sum_field, # v2\n                            answered_correctly_difficulty_weighted_sum, # v2\n                            answered_correctly_difficulty_weighted_sum_field, # v2\n                            max_solved_difficulty, # v2\n                            max_solved_difficulty_field, # v2\n                            min_wrong_difficulty, # v2\n                            min_wrong_difficulty_field, # v2\n                            session_time, # v2\n                            since_last_session_time, # v2\n                            last_session_start_time, # v2\n                            first_action_time, # v2                  \n                            )\n        gc.collect()\n        \n        del train\n        gc.collect()\n        print(n + 1, '/', n_split)\n    print('Finish train feature calculation!')\n\n    add_train_features(valid, \n                        answered_correctly_u_count, # v1 & v2 \n                        answered_correctly_u_sum, # v1 & v2\n                        elapsed_time_u_sum, # v1\n                        explanation_u_sum, # v1\n                        timestamp_u, # v1\n                        timestamp_u_incorrect, # v1\n                        latest_u_theta, # v1 & v2\n                        answered_correctly_q_count, # v1 & v2 \n                        answered_correctly_q_sum,  # v1\n                        elapsed_time_q_sum, # v1\n                        explanation_q_sum, # v1\n                        explanation_qtrue_sum, # v1\n                        explanation_qtrue_count, # v1\n                        latest_q_beta,  # v1 & v2\n                        answered_correctly_uq, # v1 & v2\n                        question_avg_sum_u, # v1\n                        task_container_sum, # v1 & v2\n                        task_container_count, # v1 & v2\n                        tags_sum, # v1\n                        tags_count, # v1\n                        answered_correctly_u_count_field, # v2\n                        answered_correctly_u_sum_field, # v2\n                        answered_correctly_difficulty_weighted_sum, # v2\n                        answered_correctly_difficulty_weighted_sum_field, # v2\n                        max_solved_difficulty, # v2\n                        max_solved_difficulty_field, # v2\n                        min_wrong_difficulty, # v2\n                        min_wrong_difficulty_field, # v2\n                        session_time, # v2\n                        since_last_session_time, # v2\n                        last_session_start_time, # v2\n                        first_action_time, # v2    \n                        )\n    print('Finish valid feature calculation!')\n    del valid\n    gc.collect()\n#     print('train: ')\n#     print(train.head())\n    \n#     print('valid: ')\n#     print(valid.head())\n    \n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect,\n        'latest_u_theta': latest_u_theta,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'explanation_qtrue_sum': explanation_qtrue_sum,\n        'explanation_qtrue_count': explanation_qtrue_count,\n        'latest_q_beta': latest_q_beta,\n        'answered_correctly_uq': answered_correctly_uq,\n        'question_avg_sum_u': question_avg_sum_u,\n        'task_container_sum': task_container_sum,\n        'task_container_count': task_container_count,\n        'tags_sum': tags_sum,\n        'tags_count': tags_count,\n        'answered_correctly_u_count_field': answered_correctly_u_count_field, #\n        'answered_correctly_u_sum_field': answered_correctly_u_sum_field, #\n        'answered_correctly_difficulty_weighted_sum': answered_correctly_difficulty_weighted_sum, #\n        'answered_correctly_difficulty_weighted_sum_field': answered_correctly_difficulty_weighted_sum_field, #\n        'max_solved_difficulty': max_solved_difficulty, #\n        'max_solved_difficulty_field': max_solved_difficulty_field, #\n        'min_wrong_difficulty': min_wrong_difficulty, #\n        'min_wrong_difficulty_field': min_wrong_difficulty_field, #\n        'session_time': session_time, #\n        'since_last_session_time': since_last_session_time, #\n        'last_session_start_time': last_session_start_time, #\n        'first_action_time': first_action_time, #\n    }\n    \n    questions_df['tags'] = questions_df['tags'].\\\n                        apply(lambda ts: [int(x) for x in str(ts).split() if x != 'nan'])\n    \n    return questions_df, prior_question_elapsed_time_mean, features_dicts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = False, n_split = 25)\n\nLGBM_model = lgb.Booster(model_file = '../input/lgbm-v921-01/lgbm_model_V921_0.1.lgb')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get feature dict\nanswered_correctly_u_count = features_dicts['answered_correctly_u_count']\nanswered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\nelapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\nexplanation_u_sum = features_dicts['explanation_u_sum']\nanswered_correctly_q_count = features_dicts['answered_correctly_q_count']\nanswered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\nelapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\nexplanation_q_sum = features_dicts['explanation_q_sum']\nexplanation_qtrue_sum = features_dicts['explanation_qtrue_sum'] #\nexplanation_qtrue_count = features_dicts['explanation_qtrue_count'] #\nanswered_correctly_uq = features_dicts['answered_correctly_uq']\ntimestamp_u = features_dicts['timestamp_u']\ntimestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\nquestion_avg_sum_u = features_dicts['question_avg_sum_u']\ntask_container_sum = features_dicts['task_container_sum']\ntask_container_count = features_dicts['task_container_count']\nlatest_u_theta = features_dicts['latest_u_theta']\nlatest_q_beta = features_dicts['latest_q_beta']\ntags_sum = features_dicts['tags_sum']\ntags_count = features_dicts['tags_count']\n\nanswered_correctly_u_count_field = features_dicts['answered_correctly_u_count_field']\nanswered_correctly_u_sum_field = features_dicts['answered_correctly_u_sum_field']\nanswered_correctly_difficulty_weighted_sum = features_dicts['answered_correctly_difficulty_weighted_sum']\nanswered_correctly_difficulty_weighted_sum_field = features_dicts['answered_correctly_difficulty_weighted_sum_field']\nmax_solved_difficulty = features_dicts['max_solved_difficulty']\nmax_solved_difficulty_field = features_dicts['max_solved_difficulty_field']\nmin_wrong_difficulty = features_dicts['min_wrong_difficulty']\nmin_wrong_difficulty_field = features_dicts['min_wrong_difficulty_field']\nsession_time = features_dicts['session_time']\nsince_last_session_time = features_dicts['since_last_session_time']\nlast_session_start_time = features_dicts['last_session_start_time']\nfirst_action_time =features_dicts['first_action_time']\n\ndel features_dicts\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\n# parameters\nMAX_SEQ = 240 # 210\nACCEPTED_USER_CONTENT_SIZE = 2 # 2\nEMBED_SIZE = 256 # 256\nBATCH_SIZE = 64+32 # 96\nDROPOUT = 0.1 # 0.1\n\n# load data\nn_skill = 13523\nprint(n_skill)\ngroup = joblib.load(\"../input/new-sakt-dataset/group.pkl.zip\")\n\n# model define\nclass FFN(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n        \n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.bn = nn.BatchNorm1d(bn_size)\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        x = self.relu(self.lr1(x))\n        x = self.bn(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n    \nclass FFN0(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN0, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.layer_normal = nn.LayerNorm(state_size) \n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        x=self.layer_normal(x)\n        return self.dropout(x)\n    \ndef future_mask(seq_length):\n    future_mask = (np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')\n    return torch.from_numpy(future_mask)\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):\n        super(TransformerBlock, self).__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_normal = nn.LayerNorm(embed_dim)\n        self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.ffn0  = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.layer_normal_2 = nn.LayerNorm(embed_dim)\n\n    def forward(self, value, key, query, att_mask):\n        att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n        att_output = self.dropout(self.layer_normal(att_output + value))\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        x = self.ffn(att_output)\n        x1 = self.ffn0(att_output)\n        x = self.dropout(self.layer_normal_2(x + x1 + att_output))\n        return x.squeeze(-1), att_weight\n    \nclass Encoder(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):\n        super(Encoder, self).__init__()\n        self.n_skill, self.embed_dim = n_skill, embed_dim\n        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion) for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x, question_ids):\n        device = x.device\n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        x = self.dropout(x + pos_x)\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = self.e_embedding(question_ids)\n        e = e.permute(1, 0, 2)\n        for layer in self.layers:\n            att_mask = future_mask(e.size(0)).to(device)\n            x, att_weight = layer(e, x, x, att_mask=att_mask)\n            x = x.permute(1, 0, 2)\n        x = x.permute(1, 0, 2)\n        return x, att_weight\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):\n        super(SAKTModel, self).__init__()\n        self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)\n        self.pred = nn.Linear(embed_dim, 1)\n        \n    def forward(self, x, question_ids):\n        x, att_weight = self.encoder(x, question_ids)\n        x = self.pred(x)\n        return x.squeeze(-1), att_weight\n    \nclass TestDataset(Dataset):\n    def __init__(self, samples, test_df, n_skill, max_seq=100):\n        super(TestDataset, self).__init__()\n        self.samples, self.user_ids, self.test_df = samples, [x for x in test_df[\"user_id\"].unique()], test_df\n        self.n_skill, self.max_seq = n_skill, max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n    \n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n        \n        user_id = test_info['user_id']\n        target_id = test_info['content_id']\n        \n        content_id_seq = np.zeros(self.max_seq, dtype=int)\n        answered_correctly_seq = np.zeros(self.max_seq, dtype=int)\n        \n        if user_id in self.samples.index:\n            content_id, answered_correctly = self.samples[user_id]\n            \n            seq_len = len(content_id)\n            \n            if seq_len >= self.max_seq:\n                content_id_seq = content_id[-self.max_seq:]\n                answered_correctly_seq = answered_correctly[-self.max_seq:]\n            else:\n                content_id_seq[-seq_len:] = content_id\n                answered_correctly_seq[-seq_len:] = answered_correctly\n                \n        x = content_id_seq[1:].copy()\n        x += (answered_correctly_seq[1:] == 1) * self.n_skill\n        \n        questions = np.append(content_id_seq[2:], [target_id])\n        \n        return x, questions\n\ndef create_model():\n    return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)\n\n# load model\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMODEL_PATH = '../input/new-sakt-dataset/sakt_model.pt'\nSAKT_model = create_model()\nSAKT_model.load_state_dict(torch.load(MODEL_PATH, map_location='cpu'))\n# SAKT_model.load_state_dict(torch.load(MODEL_PATH))\nSAKT_model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get api iterator and predictor\nenv = riiideducation.make_env()\niter_test = env.iter_test()\nset_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nw = 0.225\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in iter_test:    \n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_features(previous_test_df, \n                        answered_correctly_u_sum, \n                        answered_correctly_q_sum, \n                        timestamp_u_incorrect,\n                        explanation_qtrue_sum,\n                        task_container_sum,\n                        latest_u_theta,\n                        latest_q_beta,\n                        answered_correctly_u_count,\n                        answered_correctly_q_count,\n                        tags_sum,\n                        answered_correctly_u_sum_field,\n                        answered_correctly_difficulty_weighted_sum,\n                        answered_correctly_difficulty_weighted_sum_field,\n                        max_solved_difficulty,\n                        max_solved_difficulty_field,\n                        min_wrong_difficulty,\n                        min_wrong_difficulty_field)\n        \n        # HDKIM SAKT State Update\n#         prev_group = previous_test_df[previous_test_df['content_type_id'] == 0][['user_id', 'content_id', 'answered_correctly']].\\\n#         groupby('user_id').apply(lambda r: (\n#             r['content_id'].values,\n#             r['answered_correctly'].values))\n        \n#         for prev_user_id in prev_group.index:\n#             if prev_user_id in group.index:\n#                 group[prev_user_id] = (\n#                     np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:], \n#                     np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:]\n#                 )\n \n#             else:\n#                 group[prev_user_id] = (\n#                     prev_group[prev_user_id][0], \n#                     prev_group[prev_user_id][1]\n#                 )\n        # HDKIMHDKIM\n        \n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.\\\n                                                        fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    test_df = test_df.merge(questions_df, on = 'content_id', how = 'left')\n    previous_test_df = test_df.copy()\n    \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n    test_df[TARGET] = 1/4\n    test_df = add_features(test_df, \n                            answered_correctly_u_count, # v1 & v2 \n                            answered_correctly_u_sum, # v1 & v2\n                            elapsed_time_u_sum, # v1\n                            explanation_u_sum, # v1\n                            timestamp_u, # v1\n                            timestamp_u_incorrect, # v1\n                            latest_u_theta, # v1 & v2\n                            answered_correctly_q_count, # v1 & v2 \n                            answered_correctly_q_sum,  # v1\n                            elapsed_time_q_sum, # v1\n                            explanation_q_sum, # v1\n                            explanation_qtrue_sum, # v1\n                            explanation_qtrue_count, # v1\n                            latest_q_beta,  # v1 & v2\n                            answered_correctly_uq, # v1 & v2\n                            question_avg_sum_u, # v1\n                            task_container_sum, # v1 & v2\n                            task_container_count, # v1 & v2\n                            tags_sum, # v1\n                            tags_count, # v1\n                            answered_correctly_u_count_field, # v2\n                            answered_correctly_u_sum_field, # v2\n                            answered_correctly_difficulty_weighted_sum, # v2\n                            answered_correctly_difficulty_weighted_sum_field, # v2\n                            max_solved_difficulty, # v2\n                            max_solved_difficulty_field, # v2\n                            min_wrong_difficulty, # v2\n                            min_wrong_difficulty_field, # v2\n                            session_time, # v2\n                            since_last_session_time, # v2\n                            last_session_start_time, # v2\n                            first_action_time, # v2\n                            update = False)\n    test_df[TARGET] = LGBM_model.predict(test_df[FEATURES])\n    \n    # HDKIM SAKT\n#     test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)\n#     test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)\n    \n#     SAKT_outs = []\n\n#     for item in test_dataloader:\n#         x = item[0].to(device).long()\n#         target_id = item[1].to(device).long()\n\n#         with torch.no_grad():\n#             output, att_weight = SAKT_model(x, target_id)\n \n#         output = torch.sigmoid(output)\n#         output = output[:, -1]\n#         SAKT_outs.extend(output.view(-1).data.cpu().numpy())\n    \n#     test_df[TARGET] = test_df[TARGET] * (1 - w) + np.array(SAKT_outs) * w\n    # HDKIMHDKIM\n\n    set_predict(test_df[['row_id', TARGET]])\n\nprint('Job Done')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}