{"cells":[{"metadata":{},"cell_type":"markdown","source":"References\n* https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n* https://www.kaggle.com/its7171/cv-strategy\n* https://www.kaggle.com/ragnar123/riiid-model-lgbm\n* https://www.kaggle.com/manikanthr5/riiid-sakt-model-inference-public"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-01-04T07:38:04.62813Z","iopub.status.busy":"2021-01-04T07:38:04.627219Z","iopub.status.idle":"2021-01-04T07:38:05.984437Z","shell.execute_reply":"2021-01-04T07:38:05.983413Z"},"papermill":{"duration":1.37928,"end_time":"2021-01-04T07:38:05.984568","exception":false,"start_time":"2021-01-04T07:38:04.605288","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom sklearn.metrics import roc_auc_score\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport psutil\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T07:38:06.018222Z","iopub.status.busy":"2021-01-04T07:38:06.01747Z","iopub.status.idle":"2021-01-04T07:38:06.020803Z","shell.execute_reply":"2021-01-04T07:38:06.020121Z"},"papermill":{"duration":0.022994,"end_time":"2021-01-04T07:38:06.020916","exception":false,"start_time":"2021-01-04T07:38:05.997922","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Random seed\nSEED = 123\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-04T07:38:06.096422Z","iopub.status.busy":"2021-01-04T07:38:06.076256Z","iopub.status.idle":"2021-01-04T07:38:06.099726Z","shell.execute_reply":"2021-01-04T07:38:06.099187Z"},"papermill":{"duration":0.065972,"end_time":"2021-01-04T07:38:06.099874","exception":false,"start_time":"2021-01-04T07:38:06.033902","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops\ndef add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    # -----------------------------------------------------------------------\n    \n    for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):\n        \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n             \n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df\n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n            \n    return\n","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T07:38:06.160663Z","iopub.status.busy":"2021-01-04T07:38:06.140549Z","iopub.status.idle":"2021-01-04T07:38:06.163935Z","shell.execute_reply":"2021-01-04T07:38:06.163053Z"},"papermill":{"duration":0.051275,"end_time":"2021-01-04T07:38:06.16417","exception":false,"start_time":"2021-01-04T07:38:06.112895","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def read_and_preprocess(feature_engineering = False):\n    train_pickle = '../input/riiid-cross-validation-files/cv1_train.pickle'\n    valid_pickle = '../input/riiid-cross-validation-files/cv1_valid.pickle'\n    question_file = '../input/riiid-test-answer-prediction/questions.csv'\n\n    # Read data\n    feld_needed = ['timestamp', 'user_id', 'answered_correctly', 'content_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']\n    train = pd.read_pickle(train_pickle)[feld_needed]\n    valid = pd.read_pickle(valid_pickle)[feld_needed]\n    #--------------------------------------------------------------------\n    MAX_SEQ = 100\n    skills = train[\"content_id\"][train.content_type_id == False].unique()\n    n_skill = len(skills)\n    print(\"number skills\", len(skills))\n\n    group = train[['user_id', 'content_id', 'answered_correctly']][train.content_type_id == False].groupby('user_id').apply(lambda r: (\n                r['content_id'].values,\n                r['answered_correctly'].values))\n\n    for user_id in group.index:\n        q, qa = group[user_id]\n        if len(q)>MAX_SEQ:\n            group[user_id] = (q[-MAX_SEQ:],qa[-MAX_SEQ:])\n\n    import pickle\n    pickle.dump(group, open(\"group.pkl\", \"wb\"))\n    del group\n    gc.collect()\n    #--------------------AMMMAR------------------------------------------\n    train=train.append(valid, ignore_index = False)\n    del(valid)\n    gc.collect()\n    train = train.groupby('user_id').tail(320).reset_index(drop=True)#222\n    valid = train.groupby('user_id').tail(16)#6\n    train.drop(valid.index, inplace=True)\n    gc.collect()\n \n    #--------------------------------------------------------------------  \n    # Filter by content_type_id to discard lectures\n    train = train.loc[train.content_type_id == False].reset_index(drop = True)\n    valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)\n    # Delete some trianing data to don't have ram problems\n    if feature_engineering:\n        train = train.iloc[-34400000:] #40M\n    \n    # Changing dtype to avoid lightgbm error\n    train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False).astype('int8')\n    valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False).astype('int8')\n    \n    # Fill prior question elapsed time with the mean\n    prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()\n    train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    \n    # Merge with question dataframe\n    questions_df = pd.read_csv(question_file)\n    questions_df['part'] = questions_df['part'].astype(np.int32)\n    questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n    \n    train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    \n    # Client dictionaries\n    answered_correctly_u_count = defaultdict(int)\n    answered_correctly_u_sum = defaultdict(int)\n    elapsed_time_u_sum = defaultdict(int)\n    explanation_u_sum = defaultdict(int)\n    timestamp_u = defaultdict(list)\n    timestamp_u_incorrect = defaultdict(list)\n    \n    # Question dictionaries\n    answered_correctly_q_count = defaultdict(int)\n    answered_correctly_q_sum = defaultdict(int)\n    elapsed_time_q_sum = defaultdict(int)\n    explanation_q_sum = defaultdict(int)\n    \n    # Client Question dictionary\n    answered_correctly_uq = defaultdict(lambda: defaultdict(int))\n    \n    print('User feature calculation started...')\n    print('\\n')\n    train = add_features(train, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    valid = add_features(valid, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq)\n    gc.collect()\n    print('User feature calculation completed...')\n    print('\\n')\n    \n    features_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect\n    }\n    \n    return train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts, MAX_SEQ ,n_skill,skills","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T07:38:06.197868Z","iopub.status.busy":"2021-01-04T07:38:06.197188Z","iopub.status.idle":"2021-01-04T08:06:57.855843Z","shell.execute_reply":"2021-01-04T08:06:57.856431Z"},"papermill":{"duration":1731.678854,"end_time":"2021-01-04T08:06:57.856705","exception":false,"start_time":"2021-01-04T07:38:06.177851","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train, valid, questions_df, prior_question_elapsed_time_mean, features_dicts, MAX_SEQ ,n_skill,skills = read_and_preprocess(feature_engineering = True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T08:06:57.907955Z","iopub.status.busy":"2021-01-04T08:06:57.903373Z","iopub.status.idle":"2021-01-04T08:06:57.911711Z","shell.execute_reply":"2021-01-04T08:06:57.91104Z"},"papermill":{"duration":0.04136,"end_time":"2021-01-04T08:06:57.911843","exception":false,"start_time":"2021-01-04T08:06:57.870483","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Function for training and evaluation\ndef train_and_evaluate(train, valid, feature_engineering = False):\n    clfs = list()\n    num=1\n    TARGET = 'answered_correctly'\n    # Features to train and predict\n    FEATURES = [\n                'prior_question_elapsed_time', \n                'prior_question_had_explanation', \n                'part', \n                'answered_correctly_u_avg', \n                'elapsed_time_u_avg', \n                'explanation_u_avg',#\n                'answered_correctly_q_avg', \n                'elapsed_time_q_avg', #OK\n                'explanation_q_avg', #Ok\n                'answered_correctly_uq_count', \n                'timestamp_u_recency_1', \n                'timestamp_u_recency_2', \n                'timestamp_u_recency_3', \n                'timestamp_u_incorrect_recency'\n                 ]\n    #-------------------------------------------------------------------------\n    # Delete some training data to experiment faster\n    #if feature_engineering:\n        #train = train.sample(37000000, random_state = SEED)#15M\n    #-------------------------------------------------------------------------\n    gc.collect()\n    print(f'Traning with {train.shape[0]} rows and {len(FEATURES)} features')    \n    drop_cols = list(set(train.columns) - set(FEATURES))\n    y_train = train[TARGET]\n    y_val = valid[TARGET]\n    # Drop unnecessary columns\n    train.drop(drop_cols, axis = 1, inplace = True)\n    valid.drop(drop_cols, axis = 1, inplace = True)\n    gc.collect()\n    for i in range(0,num):######################\n        lgb_train = lgb.Dataset(train[FEATURES], y_train)\n        lgb_valid = lgb.Dataset(valid[FEATURES], y_val)\n        del train, y_train\n        gc.collect()\n\n        params= {'objective': 'binary',\n                 'metric': 'auc',\n                 'verbosity': -1,\n                 'boosting_type': 'gbdt',\n                 'feature_pre_filter': False,\n                 'learning_rate': 0.05,\n                 'lambda_l1': 0.0003344679079114925,\n                 'lambda_l2': 1.9095994876812776,\n                 'num_leaves': 433,\n                 'max_bin': 511,\n                 'feature_fraction': 1.0,\n                 'bagging_fraction': 1.0,\n                 'bagging_freq': 0,\n                 'min_child_samples': 20\n                }\n        \n        \n        paramsf = {'objective': 'binary', \n                  'seed': SEED,\n                  'metric': 'auc',\n                  'num_leaves': 433,\n                  'learning_rate': 0.05,\n                  'feature_fraction': 0.75,\n                  'bagging_freq': 10,\n                  'bagging_fraction': 0.80\n                 }\n\n        model = lgb.train(\n            params = params,\n            train_set = lgb_train,\n            num_boost_round = 10000,\n            valid_sets = [lgb_train, lgb_valid],\n            early_stopping_rounds = 10,\n            verbose_eval = 50\n        )\n        clfs.append(model)\n        print('Our Roc Auc score for the validation data is:', roc_auc_score(y_val, model.predict(valid[FEATURES])))\n\n        feature_importance = model.feature_importance()\n        feature_importance = pd.DataFrame({'Features': FEATURES, 'Importance': feature_importance}).sort_values('Importance', ascending = False)\n\n        fig = plt.figure(figsize = (10, 10))\n        fig.suptitle('Feature Importance', fontsize = 20)\n        plt.tick_params(axis = 'x', labelsize = 12)\n        plt.tick_params(axis = 'y', labelsize = 12)\n        plt.xlabel('Importance', fontsize = 15)\n        plt.ylabel('Features', fontsize = 15)\n        sns.barplot(x = feature_importance['Importance'], y = feature_importance['Features'], orient = 'h')\n        plt.show()\n    \n    return TARGET, FEATURES, model,clfs","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T08:07:01.843892Z","iopub.status.busy":"2021-01-04T08:07:01.843093Z","iopub.status.idle":"2021-01-04T13:33:32.87851Z","shell.execute_reply":"2021-01-04T13:33:32.879153Z"},"papermill":{"duration":19594.95371,"end_time":"2021-01-04T13:33:32.879525","exception":false,"start_time":"2021-01-04T08:06:57.925815","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TARGET, FEATURES, model,clfs = train_and_evaluate(train, valid, feature_engineering = True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:32.939518Z","iopub.status.busy":"2021-01-04T13:33:32.938555Z","iopub.status.idle":"2021-01-04T13:33:32.943204Z","shell.execute_reply":"2021-01-04T13:33:32.942586Z"},"papermill":{"duration":0.037422,"end_time":"2021-01-04T13:33:32.943319","exception":false,"start_time":"2021-01-04T13:33:32.905897","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(psutil.virtual_memory().percent)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:33.021481Z","iopub.status.busy":"2021-01-04T13:33:33.005586Z","iopub.status.idle":"2021-01-04T13:33:34.556041Z","shell.execute_reply":"2021-01-04T13:33:34.555209Z"},"papermill":{"duration":1.585891,"end_time":"2021-01-04T13:33:34.556174","exception":false,"start_time":"2021-01-04T13:33:32.970283","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#HDKIM SAKT\nimport psutil\nimport joblib\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nclass FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:34.615226Z","iopub.status.busy":"2021-01-04T13:33:34.614518Z","iopub.status.idle":"2021-01-04T13:33:53.048696Z","shell.execute_reply":"2021-01-04T13:33:53.04927Z"},"papermill":{"duration":18.46665,"end_time":"2021-01-04T13:33:53.049463","exception":false,"start_time":"2021-01-04T13:33:34.582813","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"skills = joblib.load(\"/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip\")\nn_skill = len(skills)\ngroup = joblib.load(\"/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:53.111809Z","iopub.status.busy":"2021-01-04T13:33:53.111086Z","iopub.status.idle":"2021-01-04T13:33:53.561833Z","shell.execute_reply":"2021-01-04T13:33:53.561142Z"},"papermill":{"duration":0.485722,"end_time":"2021-01-04T13:33:53.561951","exception":false,"start_time":"2021-01-04T13:33:53.076229","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nSAKT_model = SAKTModel(n_skill, embed_dim=128)\ntry:\n    SAKT_model.load_state_dict(torch.load(\"/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt\"))\nexcept:\n    SAKT_model.load_state_dict(torch.load(\"/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt\", map_location='cpu'))\nSAKT_model.to(device)\nSAKT_model.eval()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:53.633756Z","iopub.status.busy":"2021-01-04T13:33:53.629625Z","iopub.status.idle":"2021-01-04T13:33:53.638213Z","shell.execute_reply":"2021-01-04T13:33:53.637589Z"},"papermill":{"duration":0.048744,"end_time":"2021-01-04T13:33:53.638331","exception":false,"start_time":"2021-01-04T13:33:53.589587","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): \n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-04T13:33:53.739739Z","iopub.status.busy":"2021-01-04T13:33:53.729664Z","iopub.status.idle":"2021-01-04T13:33:54.258962Z","shell.execute_reply":"2021-01-04T13:33:54.256616Z"},"papermill":{"duration":0.586665,"end_time":"2021-01-04T13:33:54.259105","exception":false,"start_time":"2021-01-04T13:33:53.67244","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Using time series api that simulates production predictions\ndef inference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts):\n    \n    # Get feature dict\n    answered_correctly_u_count = features_dicts['answered_correctly_u_count']\n    answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\n    elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\n    explanation_u_sum = features_dicts['explanation_u_sum']\n    answered_correctly_q_count = features_dicts['answered_correctly_q_count']\n    answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\n    elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\n    explanation_q_sum = features_dicts['explanation_q_sum']\n    answered_correctly_uq = features_dicts['answered_correctly_uq']\n    timestamp_u = features_dicts['timestamp_u']\n    timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n    \n    # Get api iterator and predictor\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict\n    \n    previous_test_df = None\n    for (test_df, sample_prediction_df) in iter_test:\n        #if previous_test_df is not None:\n            \n        if (previous_test_df is not None) & (psutil.virtual_memory().percent<90): #New\n            print(psutil.virtual_memory().percent)   #New\n            previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0]) \n            previous_test_df = previous_test_df[previous_test_df[TARGET] != -1].reset_index(drop=True)       \n            previous_test_df['prior_question_had_explanation'].fillna(False, inplace=True)       \n            previous_test_df.prior_question_had_explanation=previous_test_df.prior_question_had_explanation.astype('int8')\n  \n        #-----------------------\n        #HDKIM SAKT State Update\n            prev_group = previous_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n                r['content_id'].values,\n                r['answered_correctly'].values))\n            for prev_user_id in prev_group.index:\n                prev_group_content = prev_group[prev_user_id][0]\n                prev_group_ac = prev_group[prev_user_id][1]\n                if prev_user_id in group.index:\n                    group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_content), \n                                           np.append(group[prev_user_id][1],prev_group_ac))\n                else:\n                    group[prev_user_id] = (prev_group_content,prev_group_ac)\n                if len(group[prev_user_id][0])>MAX_SEQ:\n                    new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                    new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                    group[prev_user_id] = (new_group_content,new_group_ac)\n\n        #HDKIMHDKIM\n        #-----------------------\n            update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n            \n        previous_test_df = test_df.copy()\n        test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n        #-----------------------\n        #HDKIM SAKT\n        test_dataset = TestDataset(group, test_df, skills)\n        test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n\n        SAKT_outs = []\n\n        for item in test_dataloader:\n            x = item[0].to(device).long()\n            target_id = item[1].to(device).long()\n\n            with torch.no_grad():\n                output, att_weight = SAKT_model(x, target_id)\n\n            output = torch.sigmoid(output)\n            output = output[:, -1]\n            SAKT_outs.extend(output.view(-1).data.cpu().numpy())\n    \n        #HDKIMHDKIM\n        #-----------------------\n        \n        test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n        test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')\n        test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n\n        \n        test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n        test_df[TARGET] = 0\n        test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)\n       \n        #-----------------------------\n        sub_preds = np.zeros(test_df.shape[0])\n        for i, model in enumerate(clfs, 1):\n            test_preds  = model.predict(test_df[FEATURES])\n            sub_preds += test_preds\n        #HDKIM\n        #test_df[target] = sub_preds / len(clfs) #HDKIM\n\n        lgbm_final = sub_preds / len(clfs)   \n        test_df[TARGET] = np.array(SAKT_outs) * 0.4 + lgbm_final * 0.6\n        #HDKIMHDKIM\n\n        set_predict(test_df[['row_id', TARGET]])\n        #-----------------------------\n        #test_df[TARGET] =  model.predict(test_df[FEATURES])\n        #set_predict(test_df[['row_id', TARGET]]) \n        \n    print('Thanks')\n    \ninference(TARGET, FEATURES, model, questions_df, prior_question_elapsed_time_mean, features_dicts)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}