{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\n\nimport numpy as np\nimport pandas as pd\n\nimport json\nfrom matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm\nimport time\nimport warnings\n\nimport riiideducation\nimport pickle\n\n\n# for SAKT\nimport random\nimport torch\nimport torch.nn as nn\nimport torch.nn.utils.rnn as rnn_utils\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\n\nimport joblib","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"XGBoost Predictor"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"\"\"\"This file is to serve XGBoost model trained in BQML.\"\"\"\n\nimport glob\nimport json\nimport os\nimport re\nimport numpy as np\nimport xgboost as xgb\n\n\nclass Predictor(object):\n  \"\"\"Class to feed input data into XGBoost model.\n\n  It performs both preprocessing and postprocessing on the input and output.\n  \"\"\"\n\n  def __init__(self, model, model_metadata, ohe_categorical_index_vocab,\n               mhe_categorical_index_vocab):\n    \"\"\"Initializes a Predictor for XGBoost model serving.\n\n    Args:\n      model: XGBoost model.\n      model_metadata: The metadata of the model.\n      ohe_categorical_index_vocab: Feature index to the vocabulary dictionary\n        for one hot encoded features.\n      mhe_categorical_index_vocab: Feature index to the vocabulary dictionary\n        for multi hot encoded features.\n\n    Returns:\n      A 'Predictor' instance.\n    \"\"\"\n    self._model = model\n    self._model_metadata = model_metadata\n    self._ohe_categorical_index_vocab = ohe_categorical_index_vocab\n    self._mhe_categorical_index_vocab = mhe_categorical_index_vocab\n    self._model_type = None\n    self._label_col = None\n    self._feature_name_to_index_map = {}\n    # This is to keep the order of features used in training.\n    self._feature_names = []\n    self._class_names = []\n\n  def _extract_model_metadata(self):\n    \"\"\"Extracts info from model metadata and fills member variables.\n\n    Raises:\n      ValueError: An error occurred when:\n        1. Invalid model type.\n        2. Label not found.\n        3. Features not found.\n        4. Class names not found for boosted_tree_classifier.\n        6. Feature index mismatch.\n        7. Invalid encode type for categorical features.\n    \"\"\"\n    if 'model_type' not in self._model_metadata or self._model_metadata[\n        'model_type'] not in [\n            'boosted_tree_regressor', 'boosted_tree_classifier'\n        ]:\n      raise ValueError('Invalid model_type in model_metadata')\n    self._model_type = self._model_metadata['model_type']\n    if 'label_col' not in self._model_metadata:\n      raise ValueError('label_col not found in model_metadata')\n    self._label_col = self._model_metadata['label_col']\n    if not self._model_metadata['features']:\n      raise ValueError('No feature found in model_metadata')\n    self._feature_names = self._model_metadata['feature_names']\n    if self._model_type == 'boosted_tree_classifier':\n      if 'class_names' not in self._model_metadata or not self._model_metadata[\n          'class_names']:\n        raise ValueError('No class_names found in model_metadata')\n      self._class_names = self._model_metadata['class_names']\n    for feature_index in range(len(self._feature_names)):\n      feature_name = self._feature_names[feature_index]\n      self._feature_name_to_index_map[feature_name] = feature_index\n      feature_metadata = self._model_metadata['features'][feature_name]\n      if 'encode_type' not in feature_metadata or not feature_metadata[\n          'encode_type']:\n        continue\n      elif feature_metadata['encode_type'] == 'ohe':\n        if feature_index not in self._ohe_categorical_index_vocab:\n          raise ValueError(\n              'feature_index %d missing in _ohe_categorical_index_vocab' %\n              feature_index)\n      elif feature_metadata['encode_type'] == 'mhe':\n        if feature_index not in self._mhe_categorical_index_vocab:\n          raise ValueError(\n              'feature_index %d missing in _mhe_categorical_index_vocab' %\n              feature_index)\n      else:\n        raise ValueError('Invalid encode_type %s for feature %s' %\n                         (feature_metadata['encode_type'], feature_name))\n\n  def _preprocess(self, data):\n    \"\"\"Preprocesses raw input data for prediction.\n\n    Args:\n      data: Raw input in 2d array.\n\n    Returns:\n      Preprocessed data in 2d array.\n\n    Raises:\n      ValueError: An error occurred when features in a data row are different\n      from the features in the model.\n    \"\"\"\n    self._extract_model_metadata()\n    preprocessed_data = []\n    for row_index in range(len(data)):\n      row = data[row_index]\n      sorted_data_feature_names = sorted(row.keys())\n      sorted_model_feature_names = sorted(self._feature_names)\n      if sorted_data_feature_names != sorted_model_feature_names:\n        raise ValueError(\n            'Row %d has different features %s than the model features %s' %\n            (row_index, ','.join(sorted_data_feature_names),\n             ','.join(sorted_model_feature_names)))\n      encoded_row = []\n      for feature_name in self._feature_names:\n        col = row[feature_name]\n        feature_index = self._feature_name_to_index_map[feature_name]\n        if feature_index in self._ohe_categorical_index_vocab:\n          # Label encoding.\n          vocab = self._ohe_categorical_index_vocab[feature_index]\n          col_value = str(col)\n          if col_value in vocab:\n            encoded_row.append(float(vocab.index(col_value)))\n          else:\n            # unseen category.\n            encoded_row.append(None)\n        elif feature_index in self._mhe_categorical_index_vocab:\n          # Multihot encoding.\n          vocab = self._mhe_categorical_index_vocab[feature_index]\n          mhe_list = [0.0] * len(vocab)\n          try:\n            for item in col:\n              item_value = str(item)\n              if item_value in vocab:\n                mhe_list[vocab.index(item_value)] = 1.0\n            encoded_row.extend(mhe_list)\n          except ValueError:\n            raise ValueError('The feature %s in row %d is not an array' %\n                             (feature_name, row_index))\n        else:\n          # Numerical feature.\n          try:\n            encoded_row.append(float(col))\n          except ValueError:\n            raise ValueError(\n                'The feature %s in row %d cannot be converted to float' %\n                (feature_name, row_index))\n      preprocessed_data.append(encoded_row)\n    return preprocessed_data\n\n  def predict(self, instances, **kwargs):\n    \"\"\"Performs prediction.\n\n    Args:\n      instances: A list of prediction input instances.\n      **kwargs: A dictionary of keyword args provided as additional fields on\n        the predict request body.\n\n    Returns:\n      A list of outputs containing the prediction results.\n    \"\"\"\n    del kwargs\n    encoded = self._preprocess(instances)\n    # We have to convert encoded from list to numpy array, otherwise xgb will\n    # take 0s as missing values.\n    prediction_input = xgb.DMatrix(\n        np.array(encoded).reshape((len(instances), -1)), missing=None)\n    if self._model_type == 'boosted_tree_classifier':\n      outputs = self._model.predict(prediction_input) # ntree_limit: MAX=600\n      final_outputs = []\n      for np_output in outputs:\n        output = np_output.tolist()\n        final_output = {}\n        final_output['predicted_{}'.format(\n            self._label_col)] = self._class_names[output.index(max(output))]\n        final_output['{}_values'.format(self._label_col)] = self._class_names\n        final_output['{}_probs'.format(self._label_col)] = output\n        final_outputs.append(final_output)\n      return final_outputs\n    else:\n      # Boosted tree regressor.\n      return {\n          'predicted_' + self._label_col:\n              self._model.predict(prediction_input).tolist()\n      }\n\n  @classmethod\n  def from_path(cls, model_dir, model_name=\"model.bst\", meta_name=\"model_metadata.json\"):\n    \"\"\"Creates an instance of Predictor using the given path.\n\n    Args:\n      model_dir: The local directory that contains the trained XGBoost model and\n        the assets including vocabularies and model metadata.\n\n    Returns:\n      An instance of 'Predictor'.\n    \"\"\"\n    # Keep model name the same as ml::kXgboostFinalModelFilename.\n    model_path = os.path.join(model_dir, model_name)\n    model = xgb.Booster(model_file=model_path)\n    #assets_path = os.path.join(model_dir, 'assets')\n    assets_path = model_dir\n    model_metadata_path = os.path.join(assets_path, meta_name)\n    with open(model_metadata_path) as f:\n      model_metadata = json.load(f)\n    txt_list = glob.glob(assets_path + '/*.txt')\n    ohe_categorical_index_vocab = {}\n    mhe_categorical_index_vocab = {}\n    for txt_file in txt_list:\n      ohe_feature_found = re.search(r'(\\d+).txt', txt_file)\n      mhe_feature_found = re.search(r'(\\d+)_array.txt', txt_file)\n      if ohe_feature_found:\n        feature_index = int(ohe_feature_found.group(1))\n        with open(txt_file) as f:\n          ohe_categorical_index_vocab[feature_index] = f.read().splitlines()\n      elif mhe_feature_found:\n        feature_index = int(mhe_feature_found.group(1))\n        with open(txt_file) as f:\n          mhe_categorical_index_vocab[feature_index] = f.read().splitlines()\n    return cls(model, model_metadata, ohe_categorical_index_vocab,\n               mhe_categorical_index_vocab)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"simulator"},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"class Iter_Valid(object):\n    def __init__(self, df, max_user=1000):\n        df = df.reset_index(drop=True)\n        self.df = df\n        self.user_answer = df['user_answer'].astype(str).values\n        self.answered_correctly = df['answered_correctly'].astype(str).values\n        df['prior_group_responses'] = \"[]\"\n        df['prior_group_answers_correct'] = \"[]\"\n        self.sample_df = df[df['content_type_id'] == 0][['row_id']]\n        self.sample_df['answered_correctly'] = 0\n        self.len = len(df)\n        self.user_id = df.user_id.values\n        self.task_container_id = df.task_container_id.values\n        self.content_type_id = df.content_type_id.values\n        self.max_user = max_user\n        self.current = 0\n        self.pre_user_answer_list = []\n        self.pre_answered_correctly_list = []\n\n    def __iter__(self):\n        return self\n    \n    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n        df= self.df[pre_start:self.current].copy()\n        sample_df = self.sample_df[pre_start:self.current].copy()\n        df.loc[pre_start,'prior_group_responses'] = '[' + \",\".join(self.pre_user_answer_list) + ']'\n        df.loc[pre_start,'prior_group_answers_correct'] = '[' + \",\".join(self.pre_answered_correctly_list) + ']'\n        self.pre_user_answer_list = user_answer_list\n        self.pre_answered_correctly_list = answered_correctly_list\n        return df, sample_df\n\n    def __next__(self):\n        added_user = set()\n        pre_start = self.current\n        pre_added_user = -1\n        pre_task_container_id = -1\n        pre_content_type_id = -1\n        user_answer_list = []\n        answered_correctly_list = []\n        while self.current < self.len:\n            crr_user_id = self.user_id[self.current]\n            crr_task_container_id = self.task_container_id[self.current]\n            crr_content_type_id = self.content_type_id[self.current]\n            if crr_user_id in added_user and (crr_user_id != pre_added_user or (crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)):\n                # known user(not prev user or (differnt task container and both question))\n                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            if len(added_user) == self.max_user:\n                if  crr_user_id == pre_added_user and (crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):\n                    user_answer_list.append(self.user_answer[self.current])\n                    answered_correctly_list.append(self.answered_correctly[self.current])\n                    self.current += 1\n                    continue\n                else:\n                    return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n            added_user.add(crr_user_id)\n            pre_added_user = crr_user_id\n            pre_task_container_id = crr_task_container_id\n            pre_content_type_id = crr_content_type_id\n            user_answer_list.append(self.user_answer[self.current])\n            answered_correctly_list.append(self.answered_correctly[self.current])\n            self.current += 1\n        if pre_start < self.current:\n            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n        else:\n            raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SAKT"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"MAX_SEQ = 240 # 210\nACCEPTED_USER_CONTENT_SIZE = 2 # 2\nEMBED_SIZE = 256 # 256\nBATCH_SIZE = 64+32 # 96\nDROPOUT = 0.1 # 0.1\n\nclass FFN(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN, self).__init__()\n        self.state_size = state_size\n        \n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.bn = nn.BatchNorm1d(bn_size)\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        x = self.relu(self.lr1(x))\n        x = self.bn(x)\n        x = self.lr2(x)\n        return self.dropout(x)\n    \nclass FFN0(nn.Module):\n    def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):\n        super(FFN0, self).__init__()\n        self.state_size = state_size\n\n        self.lr1 = nn.Linear(state_size, forward_expansion * state_size)\n        self.relu = nn.ReLU()\n        self.lr2 = nn.Linear(forward_expansion * state_size, state_size)\n        self.layer_normal = nn.LayerNorm(state_size) \n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, x):\n        x = self.lr1(x)\n        x = self.relu(x)\n        x = self.lr2(x)\n        x=self.layer_normal(x)\n        return self.dropout(x)\n    \ndef future_mask(seq_length):\n    future_mask = (np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')\n    return torch.from_numpy(future_mask)\n\nclass TransformerBlock(nn.Module):\n    def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):\n        super(TransformerBlock, self).__init__()\n        self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_normal = nn.LayerNorm(embed_dim)\n        self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.ffn0  = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)\n        self.layer_normal_2 = nn.LayerNorm(embed_dim)\n\n    def forward(self, value, key, query, att_mask):\n        att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)\n        att_output = self.dropout(self.layer_normal(att_output + value))\n        att_output = att_output.permute(1, 0, 2) # att_output: [s_len, bs, embed] => [bs, s_len, embed]\n        x = self.ffn(att_output)\n        x1 = self.ffn0(att_output)\n        x = self.dropout(self.layer_normal_2(x + x1 + att_output))\n        return x.squeeze(-1), att_weight\n    \nclass Encoder(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):\n        super(Encoder, self).__init__()\n        self.n_skill, self.embed_dim = n_skill, embed_dim\n        self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)\n        self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)\n        self.e_embedding = nn.Embedding(n_skill+1, embed_dim)\n        self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion) for _ in range(num_layers)])\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x, question_ids):\n        device = x.device\n        x = self.embedding(x)\n        pos_id = torch.arange(x.size(1)).unsqueeze(0).to(device)\n        pos_x = self.pos_embedding(pos_id)\n        x = self.dropout(x + pos_x)\n        x = x.permute(1, 0, 2) # x: [bs, s_len, embed] => [s_len, bs, embed]\n        e = self.e_embedding(question_ids)\n        e = e.permute(1, 0, 2)\n        for layer in self.layers:\n            att_mask = future_mask(e.size(0)).to(device)\n            x, att_weight = layer(e, x, x, att_mask=att_mask)\n            x = x.permute(1, 0, 2)\n        x = x.permute(1, 0, 2)\n        return x, att_weight\n\nclass SAKTModel(nn.Module):\n    def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):\n        super(SAKTModel, self).__init__()\n        self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)\n        self.pred = nn.Linear(embed_dim, 1)\n        \n    def forward(self, x, question_ids):\n        x, att_weight = self.encoder(x, question_ids)\n        x = self.pred(x)\n        return x.squeeze(-1), att_weight\n    \nclass TestDataset(Dataset):\n    def __init__(self, samples, test_df, n_skill, max_seq=100):\n        super(TestDataset, self).__init__()\n        self.samples, self.user_ids, self.test_df = samples, [x for x in test_df[\"user_id\"].unique()], test_df\n        self.n_skill, self.max_seq = n_skill, max_seq\n\n    def __len__(self):\n        return self.test_df.shape[0]\n    \n    def __getitem__(self, index):\n        test_info = self.test_df.iloc[index]\n        \n        user_id = test_info['user_id']\n        target_id = test_info['content_id']\n        \n        content_id_seq = np.zeros(self.max_seq, dtype=int)\n        answered_correctly_seq = np.zeros(self.max_seq, dtype=int)\n        \n        if user_id in self.samples.index:\n            content_id, answered_correctly = self.samples[user_id]\n            \n            seq_len = len(content_id)\n            \n            if seq_len >= self.max_seq:\n                content_id_seq = content_id[-self.max_seq:]\n                answered_correctly_seq = answered_correctly[-self.max_seq:]\n            else:\n                content_id_seq[-seq_len:] = content_id\n                answered_correctly_seq[-seq_len:] = answered_correctly\n                \n        x = content_id_seq[1:].copy()\n        x += (answered_correctly_seq[1:] == 1) * self.n_skill\n        \n        questions = np.append(content_id_seq[2:], [target_id])\n        \n        return x, questions\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"モデル読み込み"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/riiid-xgboost-model-and-features | grep xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    FOLD = 0\n    ROOT_PATH = \"/kaggle/input/riiid-xgboost-model-and-features\"\n    MODEL_NAME = \"xgb_v17_06_f0\"\n    validaten_flg = False\n    DDOF = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = f\"{config.ROOT_PATH}/{config.MODEL_NAME}/{config.MODEL_NAME}\"\nmodel_name = f\"{config.MODEL_NAME}_model.bst\"\nmodel_meta = f\"{config.MODEL_NAME}_assets_model_metadata.json\"\nmodel = Predictor.from_path(model_path, model_name=model_name, meta_name=model_meta)\n\nmodel._extract_model_metadata()\nfeature_names = model._feature_names\n\ncategorical_features = [feature_names[i] for i in model._ohe_categorical_index_vocab.keys()]\n\nprint(\"features:\", len(feature_names))\ncategorical_features\n\n# Check multi hot encoding features\nprint(model._mhe_categorical_index_vocab)\nassert len(model._mhe_categorical_index_vocab) == 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group = joblib.load(\"/kaggle/input/riiid-sakt-model/group.pkl.zip\")\nn_skill = joblib.load(\"/kaggle/input/riiid-sakt-model/skills.pkl.zip\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ndef create_model():\n    return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)\nsakt_model = create_model()\nsakt_model.load_state_dict(torch.load(\"/kaggle/input/riiid-sakt-model/sakt_model.pt\"))\nsakt_model.to(device)\nsakt_model.eval()\n\nsakt_model_b = create_model()\nsakt_model_b.load_state_dict(torch.load(\"/kaggle/input/riiid-sakt-model/best_sakt_model_1.pt\"))\nsakt_model_b.to(device)\nsakt_model_b.eval()\n\nprint(\"all model loaded\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"コンテンツ特徴量読み込み"},{"metadata":{"trusted":true},"cell_type":"code","source":"content_agg_feats = pd.read_csv(f\"{config.ROOT_PATH}/content_agg_feats.csv\")\nquestion_tags_ohe = pd.read_csv(f\"{config.ROOT_PATH}/question_tags_ohe.csv\")\nlecture_tags_ohe = pd.read_csv(f\"{config.ROOT_PATH}/lecture_tags_ohe.csv\")\nquestions = pd.read_csv(\"/kaggle/input/riiid-test-answer-prediction/questions.csv\")\n\nquestion_tags_ohe = question_tags_ohe.rename(columns={'question_id':'content_id'})\nlecture_tags_ohe = lecture_tags_ohe.rename(columns={'lecture_id':'content_id'})\nquestions = questions.rename(columns={'question_id':'content_id'}).drop([\"bundle_id\", \"correct_answer\", \"tags\"], axis=1)\ncontent_agg_feats = content_agg_feats.merge(question_tags_ohe, how=\"left\", on=\"content_id\")\ncontent_agg_feats = content_agg_feats.merge(questions, how=\"left\", on=\"content_id\")\ncontent_agg_feats = content_agg_feats.merge(lecture_tags_ohe, how=\"outer\", on=\"content_id\")\n\ncontent_agg_feats = content_agg_feats.fillna(0)\n\ncontent_agg_feats_v = content_agg_feats.values\ncontent_agg_feats_c = content_agg_feats.columns.values[1:]\n\nq_ohe_dic = {i: v for i, v in enumerate(question_tags_ohe.set_index(\"content_id\").values)}\nl_ohe_dic = {i: row.values for i, row in lecture_tags_ohe.drop(\"type_of\", axis=1).set_index(\"content_id\").iterrows()}\n\ndel lecture_tags_ohe, question_tags_ohe, questions, content_agg_feats\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ユーザー特徴量読み込み"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_agg_feats_even = pd.read_csv(f\"{config.ROOT_PATH}/user_agg_feat_even.csv\")\nuser_agg_feats_odd = pd.read_csv(f\"{config.ROOT_PATH}/user_agg_feat_odd.csv\")\nuser_agg_feats_df = pd.concat([user_agg_feats_even, user_agg_feats_odd])\n\nuser_agg_feats_v = user_agg_feats_df.values\n\ndel user_agg_feats_df, user_agg_feats_even, user_agg_feats_odd\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_last_timestamp = pd.read_csv(f\"{config.ROOT_PATH}/user_last_timestamp.csv\")\nlast_timestamp_dic = {k: v for k, v in user_last_timestamp.values}\ndel user_last_timestamp\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Window特徴量"},{"metadata":{"trusted":true},"cell_type":"code","source":"#WINDOW = \"/kaggle/input/riiid-create-data-for-transformer-train-on-gpu\"\nWINDOW = config.ROOT_PATH\n\nwith open(f\"{WINDOW}/user_all_count.pkl\", \"rb\") as f:\n    user_all_count = pickle.load(f)\n    \nwith open(f\"{WINDOW}/user_correct_window_200.pkl\", \"rb\") as f:\n    user_correct_window_200 = pickle.load(f)\n    \nwith open(f\"{WINDOW}/prior_question_elapsed_time_window_dict.pkl\", \"rb\") as f:\n     prior_question_elapsed_time_window_dict = pickle.load(f)\n\nwith open(f\"{WINDOW}/prior_question_had_explanation_count.pkl\", \"rb\") as f:\n     prior_question_had_explanation_count = pickle.load(f)\n        \nwith open(f\"{WINDOW}/prior_question_had_explanation_window_dict.pkl\", \"rb\") as f:\n     prior_question_had_explanation_window_dict = pickle.load(f)\n\nwith open(f\"{WINDOW}/timediff_window_dict.pkl\", \"rb\") as f:\n     timediff_window_dict = pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"カラムの設定"},{"metadata":{"trusted":true},"cell_type":"code","source":"col1 = [f\"work_q_tag_{i}_v3\" for i in range(188)]\ncol2 = [f\"cumsum_q_tag_{i}_v3\" for i in range(188)]\ncol3 = [f\"work_l_tag_{i}_v2\" for i in range(188)]\nuser_agg_feats_c = col1 + col2 + col3\n\nrate_col = [f\"correct_rate_q_tag_{i}\" for i in range(188)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"学習中の集計関数"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_content_feature(_content_id):\n    idx = np.where(content_agg_feats_v[:,0] == _content_id)[0][0]\n    v = content_agg_feats_v[idx, 1:]\n    return v.tolist()\n\ndef get_user_feature(_user_id):\n    idx = np.where(user_agg_feats_v[:,0] == _user_id)[0]\n    if len(idx) == 0:\n        return np.zeros(user_agg_feats_v.shape[1] - 1)\n    else:\n        idx = idx[0]\n        v = user_agg_feats_v[idx, 1:]\n        return v.tolist()\n    \ndef get_timediff(row):\n    _timestamp = row[\"timestamp\"]\n    _user_id = row[\"user_id\"]\n    try:\n        return _timestamp - last_timestamp_dic[_user_id]\n    except KeyError:\n        return 0\n    \n    \ndef get_lgbm_window_feat(_user_id):\n    try:\n        v = prior_question_elapsed_time_window_dict[_user_id]\n        v = np.array(v)[~np.isnan(v)]\n        prior_question_elapsed_time_std_w200 = v.std(ddof=config.DDOF)\n        prior_question_elapsed_time_avg_w200 = v.mean()\n        \n        prior_question_had_explanation_std_w200 = np.std(prior_question_had_explanation_window_dict[_user_id], ddof=config.DDOF)\n        prior_question_had_explanation_avg_w200 = np.mean(prior_question_had_explanation_window_dict[_user_id])\n        timediff_std_w200 = np.std(timediff_window_dict[_user_id], ddof=config.DDOF)\n        timediff_avg_w200 = np.mean(timediff_window_dict[_user_id])\n        _prior_question_had_explanation_count = prior_question_had_explanation_count[_user_id]\n    except KeyError:\n        prior_question_elapsed_time_std_w200 = 0\n        prior_question_elapsed_time_avg_w200 = 0\n        prior_question_had_explanation_std_w200 = 0\n        prior_question_had_explanation_avg_w200 = 0\n        timediff_std_w200 = 0\n        timediff_avg_w200 = 0\n        _prior_question_had_explanation_count = 0\n    return [\n        prior_question_elapsed_time_std_w200,\n        prior_question_elapsed_time_avg_w200,\n        prior_question_had_explanation_std_w200,\n        prior_question_had_explanation_avg_w200,\n        timediff_std_w200,\n        timediff_avg_w200,\n        _prior_question_had_explanation_count, \n    ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"学習中の状態更新関数"},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_infomation(row):\n    global user_agg_feats_v\n    \n    _user_id = row[\"user_id\"]\n    _timestamp = row[\"timestamp\"]\n    _content_id = row[\"content_id\"]\n    _answered_correctly = row[\"answered_correctly\"]\n    _content_type_id = row[\"content_type_id\"]\n    \n    try:\n        _prior_question_had_explanation = int(row[\"prior_question_had_explanation\"])\n    except TypeError:\n        _prior_question_had_explanation = 0\n        \n    try:\n        _prior_question_elapsed_time = float(row[\"prior_question_elapsed_time\"])\n    except TypeError:\n        _prior_question_elapsed_time = 0\n    \n    try:\n        _timediff = _timestamp - last_timestamp_dic[_user_id]\n    except KeyError:\n        _timediff = 0\n    \n    # timestamp update\n    last_timestamp_dic[_user_id] = _timestamp\n    \n    # get content tag values\n    if _content_type_id == 0:\n        n_work = q_ohe_dic[_content_id]\n        n_correct = n_work * _answered_correctly\n        n_lecture = np.zeros(188)\n    else:\n        n_work = np.zeros(188)\n        n_correct = np.zeros(188)\n        n_lecture = l_ohe_dic[_content_id]\n    tag_feats = np.hstack([n_work, n_correct, n_lecture])\n    \n    # user features\n    idx = np.where(user_agg_feats_v[:,0] == _user_id)[0]\n    if len(idx) == 0:\n        # append\n        append_v = np.hstack([_user_id, tag_feats]).astype(int)\n        user_agg_feats_v = np.vstack([user_agg_feats_v, append_v])\n    else:\n        # update\n        idx = idx[0]\n        user_agg_feats_v[idx, 1:] += tag_feats.astype(int)\n    \n    # count feature\n    try:\n        prior_question_had_explanation_count[_user_id] += _prior_question_had_explanation\n        user_all_count[_user_id][0] += 1\n        user_all_count[_user_id][1] += int(_answered_correctly == 1)\n    except KeyError:\n        prior_question_had_explanation_count[_user_id] = _prior_question_had_explanation\n        user_all_count[_user_id] = [1, int(_answered_correctly == 1)]\n    \n    # Window features\n    try:\n        if len(user_correct_window_200[_user_id]) == 201:\n            user_correct_window_200[_user_id].pop(0)\n            prior_question_elapsed_time_window_dict[_user_id].pop(0)\n            prior_question_had_explanation_window_dict[_user_id].pop(0)\n            timediff_window_dict[_user_id].pop(0)\n        user_correct_window_200[_user_id].append(int(_answered_correctly == 1))\n        prior_question_elapsed_time_window_dict[_user_id].append(_prior_question_elapsed_time)\n        prior_question_had_explanation_window_dict[_user_id].append(_prior_question_had_explanation)\n        timediff_window_dict[_user_id].append(_timediff)\n    except KeyError:\n        user_correct_window_200[_user_id] = [int(_answered_correctly == 1)]\n        prior_question_elapsed_time_window_dict[_user_id] = [_prior_question_elapsed_time]\n        prior_question_had_explanation_window_dict[_user_id] = [_prior_question_had_explanation]\n        timediff_window_dict[_user_id] = [_timediff]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"窓関数"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_window_n_correctry(_user_id):\n    try:\n        v = np.array(user_correct_window_200[_user_id])\n    except KeyError:\n        return [0, 0, 0]\n    _v = (v == 1).sum()\n    v_std = np.std((v == 1), ddof=config.DDOF)\n    if len(v) == 201:\n        return [_v, _v/200, v_std]\n    else:\n        return [_v, 0, v_std]\n    \ndef get_all_count(_user_id):\n    try:\n        v = user_all_count[_user_id]\n    except KeyError:\n        return [0, 0, 0]\n    \n    return v + [v[1]/v[0]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Infer"},{"metadata":{"trusted":true},"cell_type":"code","source":"if config.validaten_flg:\n    target_df = pd.read_pickle('../input/riiid-cross-validation-files/cv1_valid.pickle')\n    iter_test = Iter_Valid(target_df, max_user=1000)\n    predicted = []\n    def set_predict(df):\n        predicted.append(df)\n    user_agg_feats_v = user_agg_feats_v[:10000]\n    last_timestamp_dic = {k: last_timestamp_dic[k] for k in user_agg_feats_v[:, 0]}\n    user_correct_window_200 = {k: user_correct_window_200[k] for k in user_agg_feats_v[:, 0]}\n    user_all_count = {k: user_all_count[k] for k in user_agg_feats_v[:, 0]}\n    prior_question_had_explanation_count = {k: prior_question_had_explanation_count[k] for k in user_agg_feats_v[:, 0]}\n    user_correct_window_200 = {k: user_correct_window_200[k] for k in user_agg_feats_v[:, 0]}\n    prior_question_elapsed_time_window_dict = {k: prior_question_elapsed_time_window_dict[k] for k in user_agg_feats_v[:, 0]}\n    prior_question_had_explanation_window_dict = {k: prior_question_had_explanation_window_dict[k] for k in user_agg_feats_v[:, 0]}\n    timediff_window_dict = {k: timediff_window_dict[k] for k in user_agg_feats_v[:, 0]}\nelse:\n    env = riiideducation.make_env()\n    iter_test = env.iter_test()\n    set_predict = env.predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import psutil\nprint(psutil.virtual_memory().percent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prev_df = None\npbar = tqdm(total=2500000)\nwarnings.simplefilter('ignore')\nfor (test_df, sample_prediction_df) in iter_test:\n    if prev_df is not None:\n        prev_df[\"answered_correctly\"] = eval(test_df[\"prior_group_answers_correct\"].iloc[0])\n        _ = prev_df.apply(update_infomation, axis=1)\n        \n        # update for SAKT\n        prev_test_df = prev_df[prev_df.content_type_id == False]\n        prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id').apply(lambda r: (\n            r['content_id'].values,\n            r['answered_correctly'].values))\n        for prev_user_id in prev_group.index:\n            prev_group_content = prev_group[prev_user_id][0]\n            prev_group_answered_correctly = prev_group[prev_user_id][1]\n            if prev_user_id in group.index:\n                group[prev_user_id] = (np.append(group[prev_user_id][0], prev_group_content), \n                                       np.append(group[prev_user_id][1], prev_group_answered_correctly))\n            else:\n                group[prev_user_id] = (prev_group_content, prev_group_answered_correctly)\n            \n            if len(group[prev_user_id][0]) > MAX_SEQ:\n                new_group_content = group[prev_user_id][0][-MAX_SEQ:]\n                new_group_answered_correctly = group[prev_user_id][1][-MAX_SEQ:]\n                group[prev_user_id] = (new_group_content, new_group_answered_correctly)\n        \n    prev_df = test_df.reset_index(drop=True)\n    _test_df = test_df\n\n    # merge features contents\n    c_agg_feat_df = pd.DataFrame(_test_df[\"content_id\"].map(get_content_feature).tolist(), columns=content_agg_feats_c)\n    _test_df = pd.concat([_test_df.reset_index(drop=True), c_agg_feat_df], axis=1)\n    \n    # merge features users\n    u_agg_feat_df = pd.DataFrame(_test_df[\"user_id\"].map(get_user_feature).tolist(), columns=user_agg_feats_c)\n    _test_df = pd.concat([_test_df.reset_index(drop=True), u_agg_feat_df], axis=1)\n    \n    # calcurate rate \n    rate_df = pd.DataFrame(np.nan_to_num(_test_df[col2].values/_test_df[col1].values), columns=rate_col)\n    _test_df = pd.concat([_test_df.reset_index(drop=True), rate_df], axis=1)\n    \n    # calc timediff\n    _test_df[\"timediff\"] = _test_df.apply(lambda x: get_timediff(x), axis=1)\n    \n    # fill Nan\n    _test_df[\"prior_question_had_explanation\"] = _test_df[\"prior_question_had_explanation\"].fillna(0).astype(int)\n    \n    # Window features\n    w_df = pd.DataFrame(_test_df[\"user_id\"].map(get_window_n_correctry).tolist(),\n                        columns=[\"correct_sum_w200\", \"rate_sum_w200\", \"correct_std_w200\"])\n    all_df = pd.DataFrame(_test_df[\"user_id\"].map(get_all_count).tolist(),\n                          columns=['work_sum_all', 'correct_sum_all', 'rate_sum_all'])\n    _test_df = pd.concat([_test_df, w_df, all_df], axis=1)\n    \n    # LGBM features\n    new_feat = pd.DataFrame(_test_df[\"user_id\"].map(get_lgbm_window_feat).tolist(),\n            columns=[\n                \"prior_question_elapsed_time_std_w200\",\n                \"prior_question_elapsed_time_avg_w200\",\n                \"prior_question_had_explanation_std_w200\",\n                \"prior_question_had_explanation_avg_w200\",\n                \"timediff_std_w200\",\n                \"timediff_avg_w200\",\n                \"prior_question_had_explanation_count\"\n    ])\n    _test_df = pd.concat([_test_df, new_feat], axis=1)\n    \n    _test_df[\"work_per_time\"] = (_test_df[\"work_sum_all\"]/_test_df[\"timestamp\"]).fillna(0).values\n    _test_df[\"correct_per_time\"] = (_test_df[\"correct_sum_all\"]/_test_df[\"timestamp\"]).fillna(0).values \n    _test_df[\"prior_question_per_time\"] = (_test_df[\"prior_question_had_explanation_count\"]/_test_df[\"timestamp\"]).fillna(0).values\n    \n    # transfer feature df row to dict\n    feature_df = _test_df[feature_names].reset_index(drop=True)\n    feature_dict = [row.to_dict() for _, row in feature_df.iterrows()]\n\n    # SAKT\n    sakt_test_df = test_df[test_df.content_type_id == False]\n    \n    test_dataset = TestDataset(group, sakt_test_df, n_skill, max_seq=MAX_SEQ)\n    test_dataloader = DataLoader(test_dataset, batch_size=len(sakt_test_df), shuffle=False)\n    \n    item = next(iter(test_dataloader))\n    x = item[0].to(device).long()\n    target_id = item[1].to(device).long()\n    \n    with torch.no_grad():\n        output, _ = sakt_model(x, target_id)\n    output = torch.sigmoid(output)\n    output = output[:, -1]\n    sakt_pred = output.cpu().numpy()\n    \n    with torch.no_grad():\n        output_b, _ = sakt_model_b(x, target_id)\n    output_b = torch.sigmoid(output_b)\n    output_b = output_b[:, -1]\n    sakt_pred_b = output_b.cpu().numpy()\n      \n    # predict\n    # XGB\n    pred = model.predict(feature_dict)\n    pred = [p[\"answered_correctly_probs\"][0] for p in pred]\n    pred = [pred[i] for i, v in enumerate((test_df['content_type_id'] == 0).values) if v]\n    \n    preds_avg = np.average(np.array([pred, sakt_pred, sakt_pred_b]).T, weights=np.array([4, 1, 1]), axis=1)\n    test_df.loc[test_df['content_type_id'] == 0, \"answered_correctly\"] = preds_avg\n    \n    # Submit\n    set_predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    \n    pbar.update(len(test_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}