{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Install & Import","metadata":{}},{"cell_type":"code","source":"!pip install recbole","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import logging\nfrom logging import getLogger\nfrom recbole.config import Config\nfrom recbole.data import create_dataset, data_preparation\nfrom recbole.model.sequential_recommender import GRU4RecF, FDSA, BERT4Rec, GRU4Rec#, SASRecF\nfrom recbole.trainer import Trainer\nfrom recbole.utils import init_seed, init_logger\n\nimport random\n\nimport torch\nfrom torch import nn\n\nfrom recbole.model.abstract_recommender import SequentialRecommender\nfrom recbole.model.layers import FeedForward\n# from recbole.model.layers import FeatureSeqEmbLayer\n\nfrom recbole.utils import FeatureType\nimport copy\nimport math\n\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as fn\nfrom torch.nn.init import normal_\n\nfrom recbole.utils import FeatureType, FeatureSource\nimport torch.nn.functional as F\nfrom recbole.data.interaction import Interaction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Create atomic files for Recbole training","metadata":{}},{"cell_type":"markdown","source":"These datasets are all publicly available on kaggle. ","metadata":{}},{"cell_type":"code","source":"!mkdir /kaggle/working/hm_atomic_interation_with_item_feature\n# inter = pd.read_csv('../input/hm-atomic-interation-with-item-feature/hm_atomic_interation_with_item_feature.inter', sep='\\t')\n\ninter = pd.read_csv('../input/reduced-inter/recbox_data_post2020.inter', sep='\\t')\n# inter = inter[inter['timestamp:float'] > 1589620000 ]# 1595620000\ninter.to_csv('/kaggle/working/hm_atomic_interation_with_item_feature/hm_atomic_interation_with_item_feature.inter', index=False, sep='\\t')\ndel inter\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# item = pd.read_csv('../input/bertembedding/out_bert_embed.csv')\n# item = pd.read_csv('../input/tfidf-embedding/out_2.csv')\nitem = pd.read_csv('../input/feature-bert-embed/bert_embed_feature.csv')\nitem = item.rename(columns={'article_id':'item_id:token', 'embed': 'item_emb:float_seq'})\nprint(item.head())\nprint(item.shape)\nitem.to_csv('/kaggle/working/hm_atomic_interation_with_item_feature/hm_atomic_interation_with_item_feature.item', index=False, sep='\\t')\ndel item\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create and train Recbole model","metadata":{}},{"cell_type":"markdown","source":"## Modify encoder layers for attention weight\n\nRecBole used its own multihead attention for tansformer-based models. The code is modified to output attention weights for each layer and head.","metadata":{}},{"cell_type":"code","source":"class MultiHeadAttention(nn.Module):\n    \"\"\"\n    Multi-head Self-attention layers, a attention score dropout layer is introduced.\n    Args:\n        input_tensor (torch.Tensor): the input of the multi-head self-attention layer\n        attention_mask (torch.Tensor): the attention mask for input tensor\n    Returns:\n        hidden_states (torch.Tensor): the output of the multi-head self-attention layer\n    \"\"\"\n\n    def __init__(self, n_heads, hidden_size, hidden_dropout_prob, attn_dropout_prob, layer_norm_eps):\n        super(MultiHeadAttention, self).__init__()\n        if hidden_size % n_heads != 0:\n            raise ValueError(\n                \"The hidden size (%d) is not a multiple of the number of attention \"\n                \"heads (%d)\" % (hidden_size, n_heads)\n            )\n\n        self.num_attention_heads = n_heads\n        self.attention_head_size = int(hidden_size / n_heads)\n        self.all_head_size = self.num_attention_heads * self.attention_head_size\n        self.sqrt_attention_head_size = math.sqrt(self.attention_head_size)\n\n        self.query = nn.Linear(hidden_size, self.all_head_size)\n        self.key = nn.Linear(hidden_size, self.all_head_size)\n        self.value = nn.Linear(hidden_size, self.all_head_size)\n\n        self.softmax = nn.Softmax(dim=-1)\n        self.attn_dropout = nn.Dropout(attn_dropout_prob)\n\n        self.dense = nn.Linear(hidden_size, hidden_size)\n        self.LayerNorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)\n        self.out_dropout = nn.Dropout(hidden_dropout_prob)\n\n    def transpose_for_scores(self, x):\n        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)\n        x = x.view(*new_x_shape)\n        return x\n\n    def forward(self, input_tensor, attention_mask, need_attention=False):\n        mixed_query_layer = self.query(input_tensor)\n        mixed_key_layer = self.key(input_tensor)\n        mixed_value_layer = self.value(input_tensor)\n\n        query_layer = self.transpose_for_scores(mixed_query_layer).permute(0, 2, 1, 3)\n        key_layer = self.transpose_for_scores(mixed_key_layer).permute(0, 2, 3, 1)\n        value_layer = self.transpose_for_scores(mixed_value_layer).permute(0, 2, 1, 3)\n\n        # Take the dot product between \"query\" and \"key\" to get the raw attention scores.\n        attention_scores = torch.matmul(query_layer, key_layer)\n\n        attention_scores = attention_scores / self.sqrt_attention_head_size\n        # Apply the attention mask is (precomputed for all layers in BertModel forward() function)\n        # [batch_size heads seq_len seq_len] scores\n        # [batch_size 1 1 seq_len]\n        attention_scores = attention_scores + attention_mask\n\n        # Normalize the attention scores to probabilities.\n        attention_probs = self.softmax(attention_scores)\n        # This is actually dropping out entire tokens to attend to, which might\n        # seem a bit unusual, but is taken from the original Transformer paper.\n\n        attention_probs = self.attn_dropout(attention_probs)\n        context_layer = torch.matmul(attention_probs, value_layer)\n        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()\n        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)\n        context_layer = context_layer.view(*new_context_layer_shape)\n        hidden_states = self.dense(context_layer)\n        hidden_states = self.out_dropout(hidden_states)\n        hidden_states = self.LayerNorm(hidden_states + input_tensor)\n        if need_attention:\n            return hidden_states, attention_probs\n        else:\n            return hidden_states\n\nclass TransformerLayer(nn.Module):\n    \"\"\"\n    One transformer layer consists of a multi-head self-attention layer and a point-wise feed-forward layer.\n    Args:\n        hidden_states (torch.Tensor): the input of the multi-head self-attention sublayer\n        attention_mask (torch.Tensor): the attention mask for the multi-head self-attention sublayer\n    Returns:\n        feedforward_output (torch.Tensor): The output of the point-wise feed-forward sublayer,\n                                           is the output of the transformer layer.\n    \"\"\"\n\n    def __init__(\n        self, n_heads, hidden_size, intermediate_size, hidden_dropout_prob, attn_dropout_prob, hidden_act,\n        layer_norm_eps\n    ):\n        super(TransformerLayer, self).__init__()\n        self.multi_head_attention = MultiHeadAttention(\n            n_heads, hidden_size, hidden_dropout_prob, attn_dropout_prob, layer_norm_eps\n        )\n        self.feed_forward = FeedForward(hidden_size, intermediate_size, hidden_dropout_prob, hidden_act, layer_norm_eps)\n\n    def forward(self, hidden_states, attention_mask, need_attention=False):\n        if need_attention:\n            attention_output, attention_probs = self.multi_head_attention(hidden_states, attention_mask, need_attention)\n        else:\n            attention_output = self.multi_head_attention(hidden_states, attention_mask, need_attention)\n        feedforward_output = self.feed_forward(attention_output)\n        if need_attention:\n            return feedforward_output, attention_probs\n        else:\n            return feedforward_output\n\n\nclass TransformerEncoder(nn.Module):\n    r\"\"\" One TransformerEncoder consists of several TransformerLayers.\n    Args:\n        n_layers(num): num of transformer layers in transformer encoder. Default: 2\n        n_heads(num): num of attention heads for multi-head attention layer. Default: 2\n        hidden_size(num): the input and output hidden size. Default: 64\n        inner_size(num): the dimensionality in feed-forward layer. Default: 256\n        hidden_dropout_prob(float): probability of an element to be zeroed. Default: 0.5\n        attn_dropout_prob(float): probability of an attention score to be zeroed. Default: 0.5\n        hidden_act(str): activation function in feed-forward layer. Default: 'gelu'\n                      candidates: 'gelu', 'relu', 'swish', 'tanh', 'sigmoid'\n        layer_norm_eps(float): a value added to the denominator for numerical stability. Default: 1e-12\n    \"\"\"\n\n    def __init__(\n        self,\n        n_layers=2,\n        n_heads=2,\n        hidden_size=64,\n        inner_size=256,\n        hidden_dropout_prob=0.5,\n        attn_dropout_prob=0.5,\n        hidden_act='gelu',\n        layer_norm_eps=1e-12\n    ):\n\n        super(TransformerEncoder, self).__init__()\n        layer = TransformerLayer(\n            n_heads, hidden_size, inner_size, hidden_dropout_prob, attn_dropout_prob, hidden_act, layer_norm_eps\n        )\n        self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(n_layers)])\n\n    def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True, need_attention=False):\n        \"\"\"\n        Args:\n            hidden_states (torch.Tensor): the input of the TransformerEncoder\n            attention_mask (torch.Tensor): the attention mask for the input hidden_states\n            output_all_encoded_layers (Bool): whether output all transformer layers' output\n        Returns:\n            all_encoder_layers (list): if output_all_encoded_layers is True, return a list consists of all transformer\n            layers' output, otherwise return a list only consists of the output of last transformer layer.\n        \"\"\"\n        all_encoder_layers = []\n        all_attention_probs = []\n        for layer_module in self.layer:\n            if need_attention:\n                hidden_states, attention_probs = layer_module(hidden_states, attention_mask, need_attention)\n            else:\n                hidden_states = layer_module(hidden_states, attention_mask)\n            if output_all_encoded_layers:\n                all_encoder_layers.append(hidden_states)\n                if need_attention:\n                    all_attention_probs.append(attention_probs)\n        if not output_all_encoded_layers:\n            all_encoder_layers.append(hidden_states)\n            if need_attention:\n                    all_attention_probs.append(attention_probs)\n        if need_attention:\n            return all_encoder_layers, all_attention_probs\n        else:\n            return all_encoder_layers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modify Feature Embeddig layers\n\nThis part adds support for masking to feature embedding fuctions.","metadata":{}},{"cell_type":"code","source":"class ContextSeqEmbAbstractLayer(nn.Module):\n    \"\"\"For Deep Interest Network and feature-rich sequential recommender systems, return features embedding matrices.\"\"\"\n\n    def __init__(self):\n        super(ContextSeqEmbAbstractLayer, self).__init__()\n        self.token_field_offsets = {}\n        self.token_embedding_table = nn.ModuleDict()\n        self.float_embedding_table = nn.ModuleDict()\n        self.token_seq_embedding_table = nn.ModuleDict()\n\n        self.token_field_names = None\n        self.token_field_dims = None\n        self.float_field_names = None\n        self.float_field_dims = None\n        self.token_seq_field_names = None\n        self.token_seq_field_dims = None\n        self.num_feature_field = None\n\n    def get_fields_name_dim(self):\n        \"\"\"get user feature field and item feature field.\n        \"\"\"\n        self.token_field_names = {type: [] for type in self.types}\n        self.token_field_dims = {type: [] for type in self.types}\n        self.float_field_names = {type: [] for type in self.types}\n        self.float_field_dims = {type: [] for type in self.types}\n        self.token_seq_field_names = {type: [] for type in self.types}\n        self.token_seq_field_dims = {type: [] for type in self.types}\n        self.num_feature_field = {type: 0 for type in self.types}\n        \n\n        for type in self.types:\n            for field_name in self.field_names[type]:\n                if self.dataset.field2type[field_name] == FeatureType.TOKEN:\n                    self.token_field_names[type].append(field_name)\n                    self.token_field_dims[type].append(self.dataset.num(field_name))\n                elif self.dataset.field2type[field_name] == FeatureType.TOKEN_SEQ:\n                    self.token_seq_field_names[type].append(field_name)\n                    self.token_seq_field_dims[type].append(self.dataset.num(field_name))\n                else:\n                    self.float_field_names[type].append(field_name)\n                    self.float_field_dims[type].append(self.dataset.num(field_name))\n                self.num_feature_field[type] += 1\n                \n    def get_embedding(self):\n        \"\"\"get embedding of all features.\n        \"\"\"\n        for type in self.types:\n            if len(self.token_field_dims[type]) > 0:\n                self.token_field_offsets[type] = np.array((0, *np.cumsum(self.token_field_dims[type])[:-1]),\n                                                          dtype=np.long)\n                self.token_embedding_table[type] = FMEmbedding(\n                    self.token_field_dims[type], self.token_field_offsets[type], self.embedding_size\n                ).to(self.device)\n            if len(self.float_field_dims[type]) > 0:\n                self.float_embedding_table[type] = nn.Embedding(\n                    np.sum(self.float_field_dims[type], dtype=np.int32), self.embedding_size\n                ).to(self.device)\n            if len(self.token_seq_field_dims) > 0:\n                self.token_seq_embedding_table[type] = nn.ModuleList()\n                for token_seq_field_dim in self.token_seq_field_dims[type]:\n                    self.token_seq_embedding_table[type].append(\n                        nn.Embedding(token_seq_field_dim, self.embedding_size).to(self.device)\n                    )\n\n    def embed_float_fields(self, float_fields, type, embed=True):\n        \"\"\"Get the embedding of float fields.\n        In the following three functions(\"embed_float_fields\" \"embed_token_fields\" \"embed_token_seq_fields\")\n        when the type is user, [batch_size, max_item_length] should be recognised as [batch_size]\n        Args:\n            float_fields(torch.Tensor): [batch_size, max_item_length, num_float_field]\n            type(str): user or item\n            embed(bool): embed or not\n        Returns:\n            torch.Tensor: float fields embedding. [batch_size, max_item_length, num_float_field, embed_dim]\n        \"\"\"\n        if not embed or float_fields is None:\n            return float_fields\n\n        num_float_field = float_fields.shape[-1]\n\n       \n        # [batch_size, max_item_length, num_float_field]\n        index = torch.arange(0, num_float_field).unsqueeze(0).expand_as(float_fields).long().to(self.device)\n        # [batch_size, max_item_length, num_float_field, embed_dim]\n        float_embedding = self.float_embedding_table[type](index)\n        float_embedding = torch.mul(float_embedding, float_fields.unsqueeze(-1))\n\n        return float_embedding\n\n    def embed_token_fields(self, token_fields, type):\n        \"\"\"Get the embedding of token fields\n        Args:\n            token_fields(torch.Tensor): input, [batch_size, max_item_length, num_token_field]\n            type(str): user or item\n        Returns:\n            torch.Tensor: token fields embedding, [batch_size, max_item_length, num_token_field, embed_dim]\n        \"\"\"\n        if token_fields is None:\n            return None\n        # [batch_size, max_item_length, num_token_field, embed_dim]\n        if type == 'item':\n            embedding_shape = token_fields.shape + (-1,)\n            token_fields = token_fields.reshape(-1, token_fields.shape[-1])\n            token_embedding = self.token_embedding_table[type](token_fields)\n            token_embedding = token_embedding.view(embedding_shape)\n        else:\n            token_embedding = self.token_embedding_table[type](token_fields)\n        return token_embedding\n\n    def embed_token_seq_fields(self, token_seq_fields, type):\n        \"\"\"Get the embedding of token_seq fields.\n        Args:\n            token_seq_fields(torch.Tensor): input, [batch_size, max_item_length, seq_len]`\n            type(str): user or item\n            mode(str): mean/max/sum\n        Returns:\n            torch.Tensor: result [batch_size, max_item_length, num_token_seq_field, embed_dim]\n        \"\"\"\n        fields_result = []\n        for i, token_seq_field in enumerate(token_seq_fields):\n            embedding_table = self.token_seq_embedding_table[type][i]\n            mask = token_seq_field != 0  # [batch_size, max_item_length, seq_len]\n            mask = mask.float()\n            value_cnt = torch.sum(mask, dim=-1, keepdim=True)  # [batch_size, max_item_length, 1]\n            token_seq_embedding = embedding_table(token_seq_field)  # [batch_size, max_item_length, seq_len, embed_dim]\n            mask = mask.unsqueeze(-1).expand_as(token_seq_embedding)\n            if self.pooling_mode == 'max':\n                masked_token_seq_embedding = token_seq_embedding - (1 - mask) * 1e9\n                result = torch.max(\n                    masked_token_seq_embedding, dim=-2, keepdim=True\n                )  # [batch_size, max_item_length, 1, embed_dim]\n                result = result.values\n            elif self.pooling_mode == 'sum':\n                masked_token_seq_embedding = token_seq_embedding * mask.float()\n                result = torch.sum(\n                    masked_token_seq_embedding, dim=-2, keepdim=True\n                )  # [batch_size, max_item_length, 1, embed_dim]\n            else:\n                masked_token_seq_embedding = token_seq_embedding * mask.float()\n                result = torch.sum(masked_token_seq_embedding, dim=-2)  # [batch_size, max_item_length, embed_dim]\n                eps = torch.FloatTensor([1e-8]).to(self.device)\n                result = torch.div(result, value_cnt + eps)  # [batch_size, max_item_length, embed_dim]\n                result = result.unsqueeze(-2)  # [batch_size, max_item_length, 1, embed_dim]\n\n            fields_result.append(result)\n        if len(fields_result) == 0:\n            return None\n        else:\n            return torch.cat(fields_result, dim=-2)  # [batch_size, max_item_length, num_token_seq_field, embed_dim]\n\n    def embed_input_fields(self, user_idx, item_idx):\n        \"\"\"Get the embedding of user_idx and item_idx\n        Args:\n            user_idx(torch.Tensor): interaction['user_id']\n            item_idx(torch.Tensor): interaction['item_id_list']\n        Returns:\n            dict: embedding of user feature and item feature\n        \"\"\"\n        user_item_feat = {'user': self.user_feat, 'item': self.item_feat}\n        user_item_idx = {'user': user_idx, 'item': item_idx}\n        float_fields_embedding = {}\n        token_fields_embedding = {}\n        token_seq_fields_embedding = {}\n        sparse_embedding = {}\n        dense_embedding = {}\n\n        for type in self.types:\n            float_fields = []\n            for field_name in self.float_field_names[type]:\n                feature = user_item_feat[type][field_name][user_item_idx[type]]\n                float_fields.append(feature if len(feature.shape) == (2 + (type == 'item')) else feature.unsqueeze(-1))\n            if len(float_fields) > 0:\n                float_fields = torch.cat(float_fields, dim=-1)  # [batch_size, max_item_length, num_float_field]\n            else:\n                float_fields = None\n            # [batch_size, max_item_length, num_float_field]\n            # or [batch_size, max_item_length, num_float_field, embed_dim] or None\n            float_fields_embedding[type] = self.embed_float_fields(float_fields, type)\n\n            token_fields = []\n            for field_name in self.token_field_names[type]:\n                feature = user_item_feat[type][field_name][user_item_idx[type]]\n                token_fields.append(feature.unsqueeze(-1))\n            if len(token_fields) > 0:\n                token_fields = torch.cat(token_fields, dim=-1)  # [batch_size, max_item_length, num_token_field]\n            else:\n                token_fields = None\n            # [batch_size, max_item_length, num_token_field, embed_dim] or None\n            token_fields_embedding[type] = self.embed_token_fields(token_fields, type)\n\n            token_seq_fields = []\n            for field_name in self.token_seq_field_names[type]:\n                feature = user_item_feat[type][field_name][user_item_idx[type]]\n                token_seq_fields.append(feature)\n            # [batch_size, max_item_length, num_token_seq_field, embed_dim] or None\n            token_seq_fields_embedding[type] = self.embed_token_seq_fields(token_seq_fields, type)\n\n            if token_fields_embedding[type] is None:\n                sparse_embedding[type] = token_seq_fields_embedding[type]\n            else:\n                if token_seq_fields_embedding[type] is None:\n                    sparse_embedding[type] = token_fields_embedding[type]\n                else:\n                    sparse_embedding[type] = torch.cat([token_fields_embedding[type], token_seq_fields_embedding[type]],\n                                                       dim=-2)\n            dense_embedding[type] = float_fields_embedding[type]\n\n        # sparse_embedding[type]\n        # shape: [batch_size, max_item_length, num_token_seq_field+num_token_field, embed_dim] or None\n        # dense_embedding[type]\n        # shape: [batch_size, max_item_length, num_float_field]\n        #     or [batch_size, max_item_length, num_float_field, embed_dim] or None\n        return sparse_embedding, dense_embedding\n\n    def forward(self, user_idx, item_idx):\n        return self.embed_input_fields(user_idx, item_idx)","metadata":{"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nclass FeatureSeqEmbLayer(ContextSeqEmbAbstractLayer):\n    \"\"\"For feature-rich sequential recommenders, return item features embedding matrices according to\n    selected features.\"\"\"\n\n    def __init__(self, dataset, embedding_size, selected_features, pooling_mode, device, mask=False):\n        super(FeatureSeqEmbLayer, self).__init__()\n\n        self.device = device\n        self.embedding_size = embedding_size\n        self.dataset = dataset\n        self.user_feat = None\n        print('device', self.device)\n        self.item_feat = self.dataset.get_item_feature().to(self.device)\n        \n        if mask:\n            feat =  self.item_feat\n            new_feat = {}\n            for key in feat.interaction.keys():\n                item_ = feat.interaction.get(key)\n                if key == 'item_id':\n                    new_feat[key] = torch.cat((item_, torch.tensor(item_.shape).to(device)))\n                else:\n                    new_feat[key] = F.pad(item_, (0, 0, 0, 1))\n            self.item_feat = Interaction(new_feat)\n            \n\n        \n\n        self.field_names = {'item': selected_features}\n\n        self.types = ['item']\n        self.pooling_mode = pooling_mode\n        try:\n            assert self.pooling_mode in ['mean', 'max', 'sum']\n        except AssertionError:\n            raise AssertionError(\"Make sure 'pooling_mode' in ['mean', 'max', 'sum']!\")\n        self.get_fields_name_dim()\n        self.get_embedding()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define BERT4RecF\n\nThis part adds feature embedding layer to Bert4Rec model so that the model could utilize item feature information.","metadata":{}},{"cell_type":"code","source":"class BERT4RecF(SequentialRecommender):\n\n    def __init__(self, config, dataset):\n        super(BERT4RecF, self).__init__(config, dataset)\n\n        # load parameters info\n        self.n_layers = config['n_layers']\n        self.n_heads = config['n_heads']\n        self.hidden_size = config['hidden_size']  # same as embedding_size\n        self.inner_size = config['inner_size']  # the dimensionality in feed-forward layer\n        self.hidden_dropout_prob = config['hidden_dropout_prob']\n        self.attn_dropout_prob = config['attn_dropout_prob']\n        self.hidden_act = config['hidden_act']\n        self.layer_norm_eps = config['layer_norm_eps']\n\n        self.mask_ratio = config['mask_ratio']\n\n        self.loss_type = config['loss_type']\n        self.initializer_range = config['initializer_range']\n\n        # add feature selection parameters\n        \n        # load dataset info\n        self.mask_token = self.n_items\n        self.mask_item_length = int(self.mask_ratio * self.max_seq_length)\n\n        # define layers and loss\n        self.selected_features = config['selected_features']\n        self.pooling_mode = config['pooling_mode']\n        self.device = config['device']\n        self.num_feature_field = sum(\n            1 if dataset.field2type[field] != FeatureType.FLOAT_SEQ else dataset.num(field)\n            for field in config['selected_features']\n        )\n        \n        \n        self.item_embedding = nn.Embedding(self.n_items + 1, self.hidden_size, padding_idx=0)\n        self.position_embedding = nn.Embedding(self.max_seq_length + 1, self.hidden_size)  # add mask_token at the last\n        self.feature_embed_layer = FeatureSeqEmbLayer(\n            dataset, self.hidden_size, self.selected_features, self.pooling_mode, self.device, mask=True\n        )\n       \n\n        \n        self.trm_encoder = TransformerEncoder(\n            n_layers=self.n_layers,\n            n_heads=self.n_heads,\n            hidden_size=self.hidden_size,\n            inner_size=self.inner_size,\n            hidden_dropout_prob=self.hidden_dropout_prob,\n            attn_dropout_prob=self.attn_dropout_prob,\n            hidden_act=self.hidden_act,\n            layer_norm_eps=self.layer_norm_eps\n        )\n        \n        self.concat_layer = nn.Linear(self.hidden_size * (1 + self.num_feature_field), self.hidden_size)\n\n        self.LayerNorm = nn.LayerNorm(self.hidden_size, eps=self.layer_norm_eps)\n        self.dropout = nn.Dropout(self.hidden_dropout_prob)\n\n        # we only need compute the loss at the masked position\n        try:\n            assert self.loss_type in ['BPR', 'CE']\n        except AssertionError:\n            raise AssertionError(\"Make sure 'loss_type' in ['BPR', 'CE']!\")\n\n        # parameters initialization\n        self.apply(self._init_weights)\n        self.other_parameter_name = ['feature_embed_layer']\n#         print('num_feature_field', self.num_feature_field)\n\n    def _init_weights(self, module):\n        \"\"\" Initialize the weights \"\"\"\n        if isinstance(module, (nn.Linear, nn.Embedding)):\n            # Slightly different from the TF version which uses truncated_normal for initialization\n            # cf https://github.com/pytorch/pytorch/pull/5617\n            module.weight.data.normal_(mean=0.0, std=self.initializer_range)\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n        if isinstance(module, nn.Linear) and module.bias is not None:\n            module.bias.data.zero_()\n\n    def _neg_sample(self, item_set):\n        item = random.randint(1, self.n_items - 1)\n        while item in item_set:\n            item = random.randint(1, self.n_items - 1)\n        return item\n\n    def _padding_sequence(self, sequence, max_length):\n        pad_len = max_length - len(sequence)\n        sequence = [0] * pad_len + sequence\n        sequence = sequence[-max_length:]  # truncate according to the max_length\n        return sequence\n\n    def reconstruct_train_data(self, item_seq):\n        \"\"\"\n        Mask item sequence for training.\n        \"\"\"\n        device = item_seq.device\n        batch_size = item_seq.size(0)\n\n        sequence_instances = item_seq.cpu().numpy().tolist()\n\n        # Masked Item Prediction\n        # [B * Len]\n        masked_item_sequence = []\n        pos_items = []\n        neg_items = []\n        masked_index = []\n#         print('self.mask_token', self.mask_token)\n        for instance in sequence_instances:\n            # WE MUST USE 'copy()' HERE!\n            masked_sequence = instance.copy()\n            pos_item = []\n            neg_item = []\n            index_ids = []\n            for index_id, item in enumerate(instance):\n                # padding is 0, the sequence is end\n                if item == 0:\n                    break\n                prob = random.random()\n                if prob < self.mask_ratio:\n                    pos_item.append(item)\n                    neg_item.append(self._neg_sample(instance))\n                    masked_sequence[index_id] = self.mask_token\n                    index_ids.append(index_id)\n            \n\n            masked_item_sequence.append(masked_sequence)\n            pos_items.append(self._padding_sequence(pos_item, self.mask_item_length))\n            neg_items.append(self._padding_sequence(neg_item, self.mask_item_length))\n            masked_index.append(self._padding_sequence(index_ids, self.mask_item_length))\n\n        # [B Len]\n        masked_item_sequence = torch.tensor(masked_item_sequence, dtype=torch.long, device=device).view(batch_size, -1)\n        # [B mask_len]\n        pos_items = torch.tensor(pos_items, dtype=torch.long, device=device).view(batch_size, -1)\n        # [B mask_len]\n        neg_items = torch.tensor(neg_items, dtype=torch.long, device=device).view(batch_size, -1)\n        # [B mask_len]\n        masked_index = torch.tensor(masked_index, dtype=torch.long, device=device).view(batch_size, -1)\n        return masked_item_sequence, pos_items, neg_items, masked_index\n\n    def reconstruct_test_data(self, item_seq, item_seq_len):\n        \"\"\"\n        Add mask token at the last position according to the lengths of item_seq\n        \"\"\"\n        padding = torch.zeros(item_seq.size(0), dtype=torch.long, device=item_seq.device)  # [B]\n        item_seq = torch.cat((item_seq, padding.unsqueeze(-1)), dim=-1)  # [B max_len+1]\n        for batch_id, last_position in enumerate(item_seq_len):\n            item_seq[batch_id][last_position] = self.mask_token\n        return item_seq\n\n    def forward(self, item_seq, need_attention=False):\n        position_ids = torch.arange(item_seq.size(1), dtype=torch.long, device=item_seq.device)\n        position_ids = position_ids.unsqueeze(0).expand_as(item_seq)\n        position_embedding = self.position_embedding(position_ids)\n        \n    \n        item_emb = self.item_embedding(item_seq)\n#         print('item_emb size', item_emb.shape)\n#         print('item_seq', item_seq.shape)\n        \n        \n        sparse_embedding, dense_embedding = self.feature_embed_layer(None, item_seq)\n        sparse_embedding = sparse_embedding['item']\n        dense_embedding = dense_embedding['item']\n        # concat the sparse embedding and float embedding\n        feature_table = []\n        if sparse_embedding is not None:\n            feature_table.append(sparse_embedding)\n        if dense_embedding is not None:\n            feature_table.append(dense_embedding)\n\n        feature_table = torch.cat(feature_table, dim=-2)\n        table_shape = feature_table.shape\n        feat_num, embedding_size = table_shape[-2], table_shape[-1]\n        feature_emb = feature_table.view(table_shape[:-2] + (feat_num * embedding_size,))\n        input_concat = torch.cat((item_emb, feature_emb), -1)  # [B 1+field_num*H]\n\n        input_emb = self.concat_layer(input_concat)\n        input_emb = input_emb + position_embedding\n        input_emb = self.LayerNorm(input_emb)\n        input_emb = self.dropout(input_emb)\n        extended_attention_mask = self.get_attention_mask(item_seq, bidirectional=True)\n        if need_attention:\n            trm_output, attention_weights = self.trm_encoder(input_emb, extended_attention_mask, output_all_encoded_layers=True, need_attention=True)\n        else:\n            trm_output = self.trm_encoder(input_emb, extended_attention_mask, output_all_encoded_layers=True)\n        output = trm_output[-1]\n        if need_attention:\n            return output, attention_weights\n        else:\n            return output  # [B L H]\n\n    def multi_hot_embed(self, masked_index, max_length):\n        \"\"\"\n        For memory, we only need calculate loss for masked position.\n        Generate a multi-hot vector to indicate the masked position for masked sequence, and then is used for\n        gathering the masked position hidden representation.\n        Examples:\n            sequence: [1 2 3 4 5]\n            masked_sequence: [1 mask 3 mask 5]\n            masked_index: [1, 3]\n            max_length: 5\n            multi_hot_embed: [[0 1 0 0 0], [0 0 0 1 0]]\n        \"\"\"\n        masked_index = masked_index.view(-1)\n        multi_hot = torch.zeros(masked_index.size(0), max_length, device=masked_index.device)\n        multi_hot[torch.arange(masked_index.size(0)), masked_index] = 1\n        return multi_hot\n\n    def calculate_loss(self, interaction):\n        item_seq = interaction[self.ITEM_SEQ]\n        masked_item_seq, pos_items, neg_items, masked_index = self.reconstruct_train_data(item_seq)\n\n        seq_output = self.forward(masked_item_seq, need_attention=False)\n        pred_index_map = self.multi_hot_embed(masked_index, masked_item_seq.size(-1))  # [B*mask_len max_len]\n        # [B mask_len] -> [B mask_len max_len] multi hot\n        pred_index_map = pred_index_map.view(masked_index.size(0), masked_index.size(1), -1)  # [B mask_len max_len]\n        # [B mask_len max_len] * [B max_len H] -> [B mask_len H]\n        # only calculate loss for masked position\n        seq_output = torch.bmm(pred_index_map, seq_output)  # [B mask_len H]\n\n        if self.loss_type == 'BPR':\n            pos_items_emb = self.item_embedding(pos_items)  # [B mask_len H]\n            neg_items_emb = self.item_embedding(neg_items)  # [B mask_len H]\n            pos_score = torch.sum(seq_output * pos_items_emb, dim=-1)  # [B mask_len]\n            neg_score = torch.sum(seq_output * neg_items_emb, dim=-1)  # [B mask_len]\n            targets = (masked_index > 0).float()\n            loss = - torch.sum(torch.log(1e-14 + torch.sigmoid(pos_score - neg_score)) * targets) \\\n                   / torch.sum(targets)\n            return loss\n\n        elif self.loss_type == 'CE':\n            loss_fct = nn.CrossEntropyLoss(reduction='none')\n#             test_item_emb = self.item_embedding_transformation(self.item_embedding.weight)[:self.n_items]  # [item_num H]\n            test_item_emb = self.item_embedding.weight[:self.n_items]\n            logits = torch.matmul(seq_output, test_item_emb.transpose(0, 1))  # [B mask_len item_num]\n            targets = (masked_index > 0).float().view(-1)  # [B*mask_len]\n\n            loss = torch.sum(loss_fct(logits.view(-1, test_item_emb.size(0)), pos_items.view(-1)) * targets) \\\n                   / torch.sum(targets)\n            return loss\n        else:\n            raise NotImplementedError(\"Make sure 'loss_type' in ['BPR', 'CE']!\")\n\n    def predict(self, interaction):\n        item_seq = interaction[self.ITEM_SEQ]\n        item_seq_len = interaction[self.ITEM_SEQ_LEN]\n        test_item = interaction[self.ITEM_ID]\n        item_seq = self.reconstruct_test_data(item_seq, item_seq_len)\n        seq_output = self.forward(item_seq)\n        seq_output = self.gather_indexes(seq_output, item_seq_len)  # [B H]\n        test_item_emb = self.item_embedding(test_item)\n        scores = torch.mul(seq_output, test_item_emb).sum(dim=1)  # [B]\n        return scores\n\n    def full_sort_predict(self, interaction):\n        item_seq = interaction[self.ITEM_SEQ]\n        item_seq_len = interaction[self.ITEM_SEQ_LEN]\n        item_seq = self.reconstruct_test_data(item_seq, item_seq_len)\n        seq_output, attention_weights = self.forward(item_seq, need_attention=True)\n        print(attention_weights[0].shape)\n        seq_output = self.gather_indexes(seq_output, item_seq_len)  # [B H]\n        test_items_emb = self.item_embedding.weight[:self.n_items]  # delete masked token\n        scores = torch.matmul(seq_output, test_items_emb.transpose(0, 1))  # [B, item_num]\n        return scores, attention_weights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model training\n\nThis part trains the model","metadata":{}},{"cell_type":"code","source":"parameter_dict = {\n    'data_path': '/kaggle/working',\n    'USER_ID_FIELD': 'user_id',\n    'ITEM_ID_FIELD': 'item_id',\n    'TIME_FIELD': 'timestamp',\n    'user_inter_num_interval': \"[40,inf)\",\n    'item_inter_num_interval': \"[40,inf)\",\n    'load_col': {'inter': ['user_id', 'item_id', 'timestamp'],\n                  'item': ['item_id', 'item_emb']\n             },\n    'selected_features': ['item_emb'],\n    'neg_sampling': None,\n    'epochs': 1,\n    'train_batch_size': 256,\n    'n_layers': 2,\n    'n_heads': 2,\n    'hidden_size': 64,\n    'inner_size': 256,\n    'hidden_dropout_prob': 0.5,\n    'attn_dropout_prob': 0.5,\n    'hidden_act': 'gelu',\n    'layer_norm_eps': 1e-12,\n    'initializer_range': 0.02,\n    'mask_ratio': 0.2,\n    'loss_type': 'CE',\n    'learning_rate': 0.002,\n    'pooling_mode': 'sum',\n    'eval_args': {\n        'split': {'RS': [10, 0, 0]},\n        'group_by': 'user',\n        'order': 'TO',\n        'mode': 'full'}\n}\n\nconfig = Config(model=BERT4RecF, dataset='hm_atomic_interation_with_item_feature', config_dict=parameter_dict)\n\n# init random seed\ninit_seed(config['seed'], config['reproducibility'])\n\n# logger initialization\ninit_logger(config)\nlogger = getLogger()\n# Create handlers\nc_handler = logging.StreamHandler()\nc_handler.setLevel(logging.INFO)\nlogger.addHandler(c_handler)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = create_dataset(config)\nlogger.info(dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data, valid_data, test_data = data_preparation(config, dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # model loading and initialization\nmodel = BERT4RecF(config, train_data.dataset).to(config['device'])\nlogger.info(model)\n\n# trainer loading and initialization\ntrainer = Trainer(config, model)\n\n# model training\nbest_valid_score, best_valid_result = trainer.fit(train_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following commented code chunk was used for loading trained models.","metadata":{}},{"cell_type":"code","source":"# model_file = \"../input/onehot-bert-m/BERT4RecF-Apr-18-2022_02-18-35.pth\"\n# checkpoint = torch.load(model_file)\n# config = checkpoint['config']\n# init_seed(config['seed'], config['reproducibility'])\n# init_logger(config)\n# logger = getLogger()\n# logger.info(config)\n# model = BERT4RecF(config, train_data.dataset).to(config['device'])\n# model.load_state_dict(checkpoint['state_dict'])\n# model.load_other_parameter(checkpoint.get('other_parameter'))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Combine models\n\nThis part makes predictions and fills out the \"cold-start\" ones with 12 most frequence items.","metadata":{}},{"cell_type":"code","source":"from recbole.utils.case_study import full_sort_topk\nfrom recbole.quick_start.quick_start import load_data_and_model\n# config, model, dataset, train_data, valid_data, test_data = load_data_and_model(\n#     model_file='/kaggle/working/saved/SASRecF-Apr-05-2022_20-56-46.pth',\n# )\nexternal_user_ids = dataset.id2token(\n    dataset.uid_field, list(range(dataset.user_num)))[1:]#fist element in array is 'PAD'(default of Recbole) ->remove it ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom recbole.data.interaction import Interaction\n\ndef add_last_item(old_interaction, last_item_id, max_len=50):\n    new_seq_items = old_interaction['item_id_list'][-1]\n    if old_interaction['item_length'][-1].item() < max_len:\n        new_seq_items[old_interaction['item_length'][-1].item()] = last_item_id\n    else:\n        new_seq_items = torch.roll(new_seq_items, -1)\n        new_seq_items[-1] = last_item_id\n    return new_seq_items.view(1, len(new_seq_items))\n\ndef predict_for_all_item(external_user_id, dataset, model):\n    model.eval()\n    with torch.no_grad():\n        uid_series = dataset.token2id(dataset.uid_field, [external_user_id])\n        index = np.isin(dataset.inter_feat[dataset.uid_field].numpy(), uid_series)\n        input_interaction = dataset[index]\n        test = {\n            'item_id_list': add_last_item(input_interaction, \n                                          input_interaction['item_id'][-1].item(), model.max_seq_length),\n            'item_length': torch.tensor(\n                [input_interaction['item_length'][-1].item() + 1\n                 if input_interaction['item_length'][-1].item() < model.max_seq_length else model.max_seq_length])\n        }\n        new_inter = Interaction(test)\n        new_inter = new_inter.to(config['device'])\n        new_scores, attention = model.full_sort_predict(new_inter)\n        new_scores = new_scores.view(-1, test_data.dataset.item_num)\n        new_scores[:, 0] = -np.inf  # set scores of [pad] to -inf\n    return torch.topk(new_scores, 12)[1], attention","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"topk_items = []\nfor external_user_id in external_user_ids[112:]:\n    topk_iid_list, attention = predict_for_all_item(external_user_id, dataset, model)\n    last_topk_iid_list = topk_iid_list[-1]\n    external_item_list = dataset.id2token(dataset.iid_field, last_topk_iid_list.cpu()).tolist()\n    topk_items.append(external_item_list)\nprint(len(topk_items))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot attention scores","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\n\nfor i in range(3):\n    for j in range(2):\n        att =  attention[i][0][j].cpu().detach().numpy()\n        plt.imshow(np.array([att[i][30:] for i in range(30, att.shape[0])]))\n        plt.yticks(range(20, -1, -5))\n        plt.colorbar()\n        plt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"external_item_str = [' '.join(x) for x in topk_items]\nresult = pd.DataFrame(external_user_ids, columns=['customer_id'])\nresult['prediction'] = external_item_str\nresult.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del external_item_str\ndel topk_items\ndel external_user_ids\ndel train_data\ndel valid_data\ndel test_data\ndel model\ndel Trainer\ndel logger\ndel dataset\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reference = pd.read_csv('../input/uid-reference/reference.csv')\nreference.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.customer_id = result.customer_id.astype('int64')\nresult.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_result = pd.merge(result, reference, how='left', left_on='customer_id', right_on='new_id', indicator=False, suffixes=(\"_x\", \"\")).drop(columns=['customer_id_x', 'new_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_result = new_result[['customer_id', 'prediction']]\nnew_result.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.read_csv('../input/cold-start/submission.csv')\nsubmit_df = pd.merge(submit_df, new_result, on='customer_id', how='outer')\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = submit_df.fillna(-1)\nsubmit_df['prediction'] = submit_df.apply(\n    lambda x: x['prediction_y'] if x['prediction_y'] != -1 else x['prediction_x'], axis=1)\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = submit_df.drop(columns=['prediction_y', 'prediction_x'])\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}