{"cells":[{"metadata":{},"cell_type":"markdown","source":"###### * Base Source: https://www.kaggle.com/wangsg/a-self-attentive-model-for-knowledge-tracing\n* My First Work: https://www.kaggle.com/leadbest/sakt-self-attentive-knowledge-tracing-submitter\n\n1. Version 1: State Updates -> LB 0.765\n2. Version 3: Random Selection of User Interactions -> LB 0.768\n3. Version 6: Small Optimization -> LB 0.771?"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nimport random\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n#HDKIM\nimport random\nrandom.seed(1)\n#HDKIMHDKIM","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfeatures = [\n    #'user_id',\n    'timestamp',\n    'lagtime',\n    'lagtime_mean',\n    'content_id',\n    'task_container_id',\n    'user_lecture_cumsum',\n    'user_lecture_lv',\n    'prior_question_elapsed_time',\n    'delta_prior_question_elapsed_time',\n    'user_correctness',\n    'user_correct_cumcount',\n    'user_correct_cumsum',\n    'content_correctness',\n    'content_correctness_std',\n    'content_count',\n    'content_sum',\n    'task_container_correctness',\n    'task_container_std',\n    'task_container_sum',\n    'bundle_correctness',\n    'attempt_no',\n    'part',\n    'part_correctness_mean',\n    'part_correctness_std',\n    'tags1',\n    'tags1_correctness_mean',\n    'tags1_correctness_std',\n    'tags2',\n    'tags3',\n    'tags4',\n    'tags5',\n    'tags6',\n    'bundle_id',\n    'part_bundle_id',\n    'explanation_mean', \n    'explanation_cumsum',\n    'prior_question_had_explanation',\n#     'part_1',\n#     'part_2',\n#     'part_3',\n#     'part_4',\n#     'part_5',\n#     'part_6',\n#     'part_7',\n#     'type_of_concept',\n#     'type_of_intention',\n#     'type_of_solving_question',\n#     'type_of_starter'\n]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_lgb1 = lgb.Booster(model_file = '../input/772lgbm/model.txt')\nmodel_lgb2 = lgb.Booster(model_file = '../input/771lgbm/model.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg = pd.read_csv('../input/newpieces/user_agg.csv')\nexplanation_agg = pd.read_csv('../input/newpieces/explanation_agg.csv')\ncontent_agg = pd.read_csv('../input/newpieces/content_agg.csv')\ntask_container_agg = pd.read_csv('../input/newpieces/task_container_agg.csv')\nuser_lecture_agg = pd.read_csv('../input/newpieces/user_lecture_agg.csv')\nlagtime_agg = pd.read_csv('../input/newpieces/lagtime_agg.csv')\nattempt_no_agg = pd.read_csv('../input/newpieces/attempt_no_agg.csv')\nmax_timestamp_u = pd.read_csv('../input/riidpieces/max_timestamp_u.csv')\nquestions_df = pd.read_csv('../input/riidpieces/questions_df.csv')\nuser_prior_question_elapsed_time = pd.read_csv('../input/riidpieces/user_prior_question_elapsed_time.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_max_attempt(user_id,content_id):\n    k = (user_id,content_id)\n\n    if k in attempt_no_sum_dict.keys():\n        attempt_no_sum_dict[k]+=1\n        return attempt_no_sum_dict[k]\n\n    attempt_no_sum_dict[k] = 1\n    return attempt_no_sum_dict[k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nuser_sum_dict = user_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_count_dict = user_agg['count'].astype('int16').to_dict(defaultdict(int))\ncontent_sum_dict = content_agg['sum'].astype('int32').to_dict(defaultdict(int))\ncontent_count_dict = content_agg['count'].astype('int32').to_dict(defaultdict(int))\n\ndel user_agg\ndel content_agg\ngc.collect()\n\ntask_container_sum_dict = task_container_agg['sum'].astype('int32').to_dict(defaultdict(int))\ntask_container_count_dict = task_container_agg['count'].astype('int32').to_dict(defaultdict(int))\ntask_container_std_dict = task_container_agg['var'].astype('float16').to_dict(defaultdict(int))\n\nexplanation_sum_dict = explanation_agg['sum'].astype('int16').to_dict(defaultdict(int))\nexplanation_count_dict = explanation_agg['count'].astype('int16').to_dict(defaultdict(int))\ndel task_container_agg\ndel explanation_agg\ngc.collect()\nuser_lecture_sum_dict = user_lecture_agg['sum'].astype('int16').to_dict(defaultdict(int))\nuser_lecture_count_dict = user_lecture_agg['count'].astype('int16').to_dict(defaultdict(int))\n\nlagtime_mean_dict = lagtime_agg['mean'].astype('int32').to_dict(defaultdict(int))\n#del prior_question_elapsed_time_agg\ndel user_lecture_agg\ndel lagtime_agg\ngc.collect()\nattempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]\nattempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))\n\ndel attempt_no_agg\ngc.collect()\nmax_timestamp_u_dict=max_timestamp_u.set_index('user_id').to_dict()\nuser_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id').to_dict()\n#del question_elapsed_time_agg\ndel max_timestamp_u\ndel user_prior_question_elapsed_time\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SKAT"},{"metadata":{"trusted":true},"cell_type":"code","source":"#HDKIM\nMAX_SEQ = 160\n#HDKIMHDKIM\ndtype = {'timestamp':'int64', \n         'user_id':'int32' ,\n         'content_id':'int16',\n         'content_type_id':'int8',\n         'answered_correctly':'int8'}\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols=[1, 2, 3, 4, 7], dtype=dtype)\ntrain_df.head()\ntrain_df = train_df[train_df.content_type_id == False]\n\n#arrange by timestamp\ntrain_df = train_df.sort_values(['timestamp'], ascending=True).reset_index(drop = True)\nskills = train_df[\"content_id\"].unique()\nn_skill = len(skills)\nprint(\"number skills\", len(skills))\ngroup = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n\ndel train_df\ngc.collect()\nclass FFN(nn.Module):\n    def __init__(self, state_size=200):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(state_size, state_size)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n\ndef future_mask(seq_length):\n    future_mask = np.triu(np.ones((seq_length, seq_length)), k=1).astype('bool')\n    return torch.from_numpy(future_mask)\n\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128): #HDKIM 100->MAX_SEQ\n        super(SAKTModel, self).__init__()\n        self.n_skill = n_skill\n        self.embed_dim = embed_dim\n\n        self.embedding = nn.Embedding(2*n_skill+1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)\n\n        self.dropout = nn.Dropout(0.2)\n        self.layer_normal = nn.LayerNorm(embed_dim) \n\n        self.ffn = FFN(embed_dim)\n        self.pred = nn.Linear(embed_dim, 1)\n    \n    def forward(self, x, question_ids):\n        device = x.device        \n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n\n        pos_x = self.pos_embedding(pos_id)\n        x = x + pos_x\n\n        e = self.e_embedding(question_ids)\n\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = e.permute(1, 0, 2)\n        att_mask = future_mask(x.size(0)).to(device)\n        att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)\n        att_output = self.layer_normal(att_output + e)\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n\n        x = self.ffn(att_output)\n        x = self.layer_normal(x + att_output)\n        x = self.pred(x)\n\n        return x.squeeze(-1), att_weight\n    \ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel_SAKT = SAKTModel(n_skill, embed_dim=128)\nmodel_SAKT.load_state_dict(torch.load('../input/771sakt/SAKT-HDKIM.pt'))\nmodel_SAKT.to(device)\nclass TestDataset(Dataset):\n    def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ): #HDKIM 100\n        super(TestDataset, self).__init__()\n        self.samples = samples\n        self.user_ids = [x for x in test_df[\"user_id\"].unique()]\n        self.test_df = test_df\n        self.skills = skills\n        self.n_skill = len(skills)\n        self.max_seq = max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n\n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n\n        user_id = test_info[\"user_id\"]\n        target_id = test_info[\"content_id\"]\n\n        q = np.zeros(self.max_seq, dtype=int)\n        qa = np.zeros(self.max_seq, dtype=int)\n\n        if user_id in self.samples.index:\n            q_, qa_ = self.samples[user_id]\n            \n            seq_len = len(q_)\n\n            if seq_len >= self.max_seq:\n                q = q_[-self.max_seq:]\n                qa = qa_[-self.max_seq:]\n            else:\n                q[-seq_len:] = q_\n                qa[-seq_len:] = qa_          \n        \n        x = np.zeros(self.max_seq-1, dtype=int)\n        x = q[1:].copy()\n        x += (qa[1:] == 1) * self.n_skill\n        \n        questions = np.append(q[2:], [target_id])\n        \n        return x, questions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGB 2"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):\n    # -----------------------------------------------------------------------\n    # Client features\n    answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_u_avg = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\n    timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # Question features\n    answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\n    elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\n    explanation_q_avg = np.zeros(len(df), dtype = np.float32)\n    # -----------------------------------------------------------------------\n    # User Question\n    answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n    # -----------------------------------------------------------------------\n    \n    for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):\n        \n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[row[0]] != 0:\n            answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n            explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n        else:\n            answered_correctly_u_avg[num] = np.nan\n            elapsed_time_u_avg[num] = np.nan\n            explanation_u_avg[num] = np.nan\n            \n        if len(timestamp_u[row[0]]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n            timestamp_u_recency_3[num] = np.nan\n        elif len(timestamp_u[row[0]]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n        \n        if len(timestamp_u_incorrect[row[0]]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]\n            \n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n        # ------------------------------------------------------------------\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[row[0]] += 1\n        elapsed_time_u_sum[row[0]] += row[3]\n        explanation_u_sum[row[0]] += int(row[4])\n        if len(timestamp_u[row[0]]) == 3:\n            timestamp_u[row[0]].pop(0)\n            timestamp_u[row[0]].append(row[5])\n        else:\n            timestamp_u[row[0]].append(row[5])\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[row[0]][row[2]] += 1\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        if update:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[5])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[5])\n            \n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n             \n            \n    user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg, \n                            'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg, \n                            'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,\n                            'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})\n    \n    df = pd.concat([df, user_df], axis = 1)\n    return df\n        \ndef update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):\n    for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:\n        if row[3] == 0:\n            # ------------------------------------------------------------------\n            # Client features updates\n            answered_correctly_u_sum[row[0]] += row[1]\n            if row[1] == 0:\n                if len(timestamp_u_incorrect[row[0]]) == 1:\n                    timestamp_u_incorrect[row[0]].pop(0)\n                    timestamp_u_incorrect[row[0]].append(row[4])\n                else:\n                    timestamp_u_incorrect[row[0]].append(row[4])\n            # ------------------------------------------------------------------\n            # Question features updates\n            answered_correctly_q_sum[row[2]] += row[1]\n            # ------------------------------------------------------------------\n            \n    return\nTARGET = 'answered_correctly'\n# Features to train and predict\nFEATURES = ['prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'answered_correctly_u_avg', 'elapsed_time_u_avg', 'explanation_u_avg',\n            'answered_correctly_q_avg', 'elapsed_time_q_avg', 'explanation_q_avg', 'answered_correctly_uq_count', 'timestamp_u_recency_1', 'timestamp_u_recency_2', 'timestamp_u_recency_3', \n            'timestamp_u_incorrect_recency']\nmodel_lgb2 = lgb.Booster(model_file = '../input/771lgbm/model.txt')\nquestions_df = pd.read_csv('../input/771lgbm/questions_df.csv')\nlgb.plot_importance(model_lgb2, importance_type='gain')\nfrom collections import defaultdict\n\nanswered_correctly_u_count = defaultdict(int)\nanswered_correctly_u_sum = defaultdict(int)\nelapsed_time_u_sum = defaultdict(int)\nexplanation_u_sum = defaultdict(int)\ntimestamp_u = defaultdict(list)\ntimestamp_u_incorrect = defaultdict(list)\n\n# Question dictionaries\nanswered_correctly_q_count = defaultdict(int)\nanswered_correctly_q_sum = defaultdict(int)\nelapsed_time_q_sum = defaultdict(int)\nexplanation_q_sum = defaultdict(int)\n\n# Client Question dictionary\nanswered_correctly_uq = defaultdict(lambda: defaultdict(int))\nfeatures_dicts = {\n        'answered_correctly_u_count': answered_correctly_u_count,\n        'answered_correctly_u_sum': answered_correctly_u_sum,\n        'elapsed_time_u_sum': elapsed_time_u_sum,\n        'explanation_u_sum': explanation_u_sum,\n        'answered_correctly_q_count': answered_correctly_q_count,\n        'answered_correctly_q_sum': answered_correctly_q_sum,\n        'elapsed_time_q_sum': elapsed_time_q_sum,\n        'explanation_q_sum': explanation_q_sum,\n        'answered_correctly_uq': answered_correctly_uq,\n        'timestamp_u': timestamp_u,\n        'timestamp_u_incorrect': timestamp_u_incorrect\n    }\nimport psutil\n#model.eval()\nanswered_correctly_u_count = features_dicts['answered_correctly_u_count']\nanswered_correctly_u_sum = features_dicts['answered_correctly_u_sum']\nelapsed_time_u_sum = features_dicts['elapsed_time_u_sum']\nexplanation_u_sum = features_dicts['explanation_u_sum']\nanswered_correctly_q_count = features_dicts['answered_correctly_q_count']\nanswered_correctly_q_sum = features_dicts['answered_correctly_q_sum']\nelapsed_time_q_sum = features_dicts['elapsed_time_q_sum']\nexplanation_q_sum = features_dicts['explanation_q_sum']\nanswered_correctly_uq = features_dicts['answered_correctly_uq']\ntimestamp_u = features_dicts['timestamp_u']\ntimestamp_u_incorrect = features_dicts['timestamp_u_incorrect']\n\n# Get api iterator and predictor\n\n#print(prior_question_elapsed_time_mean)\nprevious_test_df = None\nprior_test_df  =  None\n#HDKIM\ntarget = 'answered_correctly'\nprev_test_df = None\n#HDKIMHDKIM\nprior_question_elapsed_time_mean = 13005.0810546875","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Test"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 0:\n    prior_df = add_answers_to_prior_df(current_df,prior_df)\n    question_df = build_question_df(prior_df,question_df)\n    user_df = build_user_df(prior_df,user_df,question_df)\n(test_df, sample_prediction_df)\ncurrent_df = test_df.copy()\n#current_df = data_transform(current_df,False,False)\ncurrent_df['answered_correctly'] = 0\nenv.predict(current_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\n#model.eval()\nprior_test_df  =  None\n#HDKIM\ntarget = 'answered_correctly'\nprev_test_df = None\n#HDKIMHDKIM\nprior_question_elapsed_time_mean = 13005.0810546875\nprevious_test_df = None\nfor (test_df, sample_prediction_df) in tqdm(iter_test):\n    cur =  (test_df, sample_prediction_df) \n    QUestions_df = pd.read_csv('../input/771lgbm/questions_df.csv')\n    if previous_test_df is not None:\n        previous_test_df[TARGET] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)\n    previous_test_df = test_df.copy()\n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)\n    test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False).astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)\n    test_df = pd.merge(test_df, QUestions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')\n    test_df[TARGET] = 0\n    test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)\n    test_preds0 =  model_lgb2.predict(test_df[FEATURES])\n\n    \n    #3######################     LGBM lb 771\n    (test_df, sample_prediction_df) = cur\n\n    \n    if prior_test_df is not None:\n        prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)       \n        prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)       \n        prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')\n    \n        user_ids = prior_test_df['user_id'].values\n        content_ids = prior_test_df['content_id'].values\n        task_container_ids = prior_test_df['task_container_id'].values\n        prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values\n        targets = prior_test_df[target].values\n       \n        \n        \n        for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):\n            user_sum_dict[user_id] += answered_correctly\n            user_count_dict[user_id] += 1         \n            explanation_sum_dict[user_id] += prior_question_had_explanation\n            explanation_count_dict[user_id] += 1\n            \n    \n    prior_test_df = test_df.copy()\n    lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)\n    for i, (user_id,content_type_id, content_id) in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)):\n      \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        #\n        if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):\n            user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)\n            user_lecture_stats_part.fillna(0, inplace=True)\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n        else:\n            user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values\n  \n        \n    test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)\n   \n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')\n    test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)\n    \n\n    user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_lecture_count = np.zeros(len(test_df), dtype=np.int16) \n    \n    user_sum = np.zeros(len(test_df), dtype=np.int16)\n    user_count = np.zeros(len(test_df), dtype=np.int16)\n    content_sum = np.zeros(len(test_df), dtype=np.int32)\n    content_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_sum = np.zeros(len(test_df), dtype=np.int32)\n    task_container_count = np.zeros(len(test_df), dtype=np.int32)\n    task_container_std = np.zeros(len(test_df), dtype=np.float16)\n    content_task_mean = np.zeros(len(test_df), dtype=np.float16)\n    explanation_sum = np.zeros(len(test_df), dtype=np.int32)\n    explanation_count = np.zeros(len(test_df), dtype=np.int32)\n    delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)\n\n    attempt_no_count = np.zeros(len(test_df), dtype=np.int16)\n    lagtime = np.zeros(len(test_df), dtype=np.int32)\n    lagtime_mean = np.zeros(len(test_df), dtype=np.int32)\n   \n    prior_question_elapsed_time_mean = 13005.0810546875\n\n\n    for i, (user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id) in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)):\n         \n        user_lecture_sum_dict[user_id] += content_type_id\n        user_lecture_count_dict[user_id] += 1\n        \n        user_lecture_sum[i] = user_lecture_sum_dict[user_id]\n        user_lecture_count[i] = user_lecture_count_dict[user_id]\n        \n        user_sum[i] = user_sum_dict[user_id]\n        user_count[i] = user_count_dict[user_id]\n        content_sum[i] = content_sum_dict[content_id]\n        content_count[i] = content_count_dict[content_id]\n        task_container_sum[i] = task_container_sum_dict[task_container_id]\n        task_container_count[i] = task_container_count_dict[task_container_id]\n        task_container_std[i]=task_container_std_dict[task_container_id]\n      \n        explanation_sum[i] = explanation_sum_dict[user_id]\n        explanation_count[i] = explanation_count_dict[user_id]\n  \n        if user_id in max_timestamp_u_dict['max_time_stamp'].keys():\n            lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]\n            max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2           \n        else:\n            lagtime[i]=0\n            max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})\n            lagtime_mean_dict.update({user_id:timestamp})\n            lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2\n            \n        if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys():            \n            delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]\n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time\n        else:           \n            delta_prior_question_elapsed_time[i]=0    \n            user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})\n           \n        \n        \n    questions_df = pd.read_csv('../input/772lgbm/questions_df.csv')\n    #\n    #test_df = pd.merge(test_df, questions_df, on='content_id', how='left',right_index=True)    \n    #test_df = pd.concat([test_df.reset_index(drop=True), questions_df.reindex(test_df['content_id'].values).reset_index(drop=True)], axis=1)\n    test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],\n                  how='left', on='content_id', right_index=True)\n    \n    #test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how=\"left\",right_index=True)\n    #test_df = pd.concat([test_df.reset_index(drop=True), user_lecture_stats_part.reindex(test_df['user_id'].values).reset_index(drop=True)], axis=1)\n#     test_df=test_df.merge(user_lecture_stats_part.loc[user_lecture_stats_part.index.isin(test_df['user_id'])],\n#                   how='left', on='user_id', right_index=True)\n \n    test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count\n    test_df['user_lecture_cumsum'] = user_lecture_sum\n    test_df['user_correctness'] = user_sum / user_count\n    test_df['user_correct_cumcount'] =user_count\n    test_df['user_correct_cumsum'] =user_sum\n    #\n    test_df['content_correctness'] = content_sum / content_count\n    test_df['content_count'] = content_count\n    test_df['content_sum'] = content_sum\n    \n    test_df['task_container_correctness'] = task_container_sum / task_container_count\n    test_df['task_container_sum'] = task_container_sum \n    test_df['task_container_std'] = task_container_std \n    #test_df['content_task_mean'] = content_task_mean \n    \n    test_df['explanation_mean'] = explanation_sum / explanation_count\n    test_df['explanation_cumsum'] = explanation_sum \n    \n    #\n    test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time \n    \n  \n \n    test_df[\"attempt_no\"] = test_df[[\"user_id\", \"content_id\"]].apply(lambda row: get_max_attempt(row[\"user_id\"], row[\"content_id\"]), axis=1)\n    test_df[\"lagtime\"]=lagtime\n    test_df[\"lagtime_mean\"]=lagtime_mean\n\n    test_df['user_correctness'].fillna( 1, inplace=True)\n    test_df['attempt_no'].fillna(1, inplace=True)\n    #\n    test_df.fillna(0, inplace=True)\n    \n\n    test_df['timestamp']=test_df['timestamp']/(1000*3600)\n    test_df.timestamp=test_df.timestamp.astype('int16')\n\n\n    sub_preds = np.zeros(test_df.shape[0])\n    test_preds1  = model_lgb1.predict(test_df[features])\n    \n    \n    ####. SKAT lb 772\n\n    \n    (test_df, sample_prediction_df) = cur    \n    if (prev_test_df is not None) & (psutil.virtual_memory().percent<90):\n        print(psutil.virtual_memory().percent)\n        prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])\n        prev_test_df = prev_test_df[prev_test_df.content_type_id == False]\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_ac = prev_group[prev_user_id][1]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0],prev_group_content), \n                                       np.append(group[prev_user_id][1],prev_group_ac))\n \n            else:\n                group[prev_user_id] = (prev_group_content,prev_group_ac)\n            if len(group[prev_user_id][0])>MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_ac = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content,new_group_ac)\n\n    prev_test_df = test_df.copy()\n     \n    test_df = test_df[test_df.content_type_id == False]\n                \n    test_dataset = TestDataset(group, test_df, skills)\n    test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)\n    \n    test_preds2 = []\n\n    for item in tqdm(test_dataloader):\n        x = item[0].to(device).long()\n        target_id = item[1].to(device).long()\n\n        with torch.no_grad():\n            output, att_weight = model_SAKT(x, target_id)\n        \n        \n        output = torch.sigmoid(output)\n        output = output[:, -1]\n\n        # pred = (output >= 0.5).long()\n        # loss = criterion(output, label)\n\n        # val_loss.append(loss.item())\n        # num_corrects += (pred == label).sum().item()\n        # num_total += len(label)\n\n        # labels.extend(label.squeeze(-1).data.cpu().numpy())\n        test_preds2.extend(output.view(-1).data.cpu().numpy())\n        \n        \n        \n        \n    test_df['answered_correctly'] =  (np.array(test_preds2)+test_preds1+test_preds0)/3\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}