{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pickle\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport torch\nimport torch.nn as nn\n\n\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-21T07:55:18.449019Z","iopub.execute_input":"2022-04-21T07:55:18.451234Z","iopub.status.idle":"2022-04-21T07:55:20.735332Z","shell.execute_reply.started":"2022-04-21T07:55:18.449704Z","shell.execute_reply":"2022-04-21T07:55:20.734336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:20.739362Z","iopub.execute_input":"2022-04-21T07:55:20.739649Z","iopub.status.idle":"2022-04-21T07:55:20.748911Z","shell.execute_reply.started":"2022-04-21T07:55:20.739615Z","shell.execute_reply":"2022-04-21T07:55:20.747905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_obj(filepath):\n    obj=None\n    with open(filepath, 'rb') as file:\n        obj = pickle.load(file)\n    return obj\n\ndef save_obj(obj, filepath):\n    with open(filepath, 'wb') as file:\n        pickle.dump(obj, file)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:20.750054Z","iopub.execute_input":"2022-04-21T07:55:20.750398Z","iopub.status.idle":"2022-04-21T07:55:20.760868Z","shell.execute_reply.started":"2022-04-21T07:55:20.75035Z","shell.execute_reply":"2022-04-21T07:55:20.759945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article2id=load_obj('../input/hm-recformer-dataset/article2id.pkl')\nid2article=load_obj('../input/hm-recformer-dataset/id2article.pkl')\narticle_info = load_obj('../input/hm-recformer-dataset/article_info.pkl')\ntrain_articles = load_obj('../input/hm-recformer-dataset/train_articles.pkl')\n\ntransaction_df = pd.read_pickle(\"../input/hm-recformer-dataset/train_transaction_df.pkl\")\narticle_df = pd.read_pickle(\"../input/hm-recformer-dataset/train_article_df.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:20.76256Z","iopub.execute_input":"2022-04-21T07:55:20.763262Z","iopub.status.idle":"2022-04-21T07:55:48.342379Z","shell.execute_reply.started":"2022-04-21T07:55:20.763218Z","shell.execute_reply":"2022-04-21T07:55:48.341431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntransaction_df['has_val'] = transaction_df.train_val_split.apply( lambda data: data['val_data']['week_relative'][0] > 46 )\ntransaction_df['num_weeks'] = transaction_df.train_val_split.apply( lambda data: len(np.unique(data['train_data']['week_relative'] )) )\n\ntransaction_df.has_val.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:48.343624Z","iopub.execute_input":"2022-04-21T07:55:48.343857Z","iopub.status.idle":"2022-04-21T07:55:54.528504Z","shell.execute_reply.started":"2022-04-21T07:55:48.343829Z","shell.execute_reply":"2022-04-21T07:55:54.527728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transaction_df['num_weeks'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.529687Z","iopub.execute_input":"2022-04-21T07:55:54.529896Z","iopub.status.idle":"2022-04-21T07:55:54.542552Z","shell.execute_reply.started":"2022-04-21T07:55:54.52987Z","shell.execute_reply":"2022-04-21T07:55:54.541638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of articles:\", len(article2id))\nprint(\"number of customers:\", len(transaction_df))\nprint(\"number of train articles:\", len(train_articles))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.543782Z","iopub.execute_input":"2022-04-21T07:55:54.544047Z","iopub.status.idle":"2022-04-21T07:55:54.554261Z","shell.execute_reply.started":"2022-04-21T07:55:54.544008Z","shell.execute_reply":"2022-04-21T07:55:54.553463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.555208Z","iopub.execute_input":"2022-04-21T07:55:54.556143Z","iopub.status.idle":"2022-04-21T07:55:54.580232Z","shell.execute_reply.started":"2022-04-21T07:55:54.556101Z","shell.execute_reply":"2022-04-21T07:55:54.579556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_encoder(lst):\n    label2id={}\n    id2label={}\n    \n    for i,label in enumerate(lst):\n        label2id[label] = i\n        id2label[i] = label\n    return (label2id, id2label)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.58208Z","iopub.execute_input":"2022-04-21T07:55:54.582339Z","iopub.status.idle":"2022-04-21T07:55:54.588042Z","shell.execute_reply.started":"2022-04-21T07:55:54.582306Z","shell.execute_reply":"2022-04-21T07:55:54.587082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(section2id, id2section) = label_encoder(article_df.section_name.unique())\n(prodgroup2id, id2prodgroup) = label_encoder(article_df.product_group_name.unique())\n\nsave_obj(section2id, 'section2id.pkl')\nsave_obj(id2section, 'id2section.pkl')\n\nsave_obj(prodgroup2id, 'prodgroup2id.pkl')\nsave_obj(id2prodgroup, 'id2prodgroup.pkl')\n\nprint(len(section2id), len(prodgroup2id))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.591794Z","iopub.execute_input":"2022-04-21T07:55:54.592145Z","iopub.status.idle":"2022-04-21T07:55:54.617288Z","shell.execute_reply.started":"2022-04-21T07:55:54.592091Z","shell.execute_reply":"2022-04-21T07:55:54.616193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    NUM_EPOCHS=25\n    NUM_NEGATIVE_SAMPLES = 100\n    MAX_SEQ_LEN = 64\n    BATCH_SIZE=1024\n    \n    MAX_TARGETS = 6\n    MAX_TIME_ID = 51\n    PAD_TIME_ID = 52\n    \n    PAD_ARTICLE_ID = len(article2id)\n    PAD_SECTION_ID = article_df.section_name.nunique()\n    PAD_PRODUCT_GROUP_ID = article_df.product_group_name.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.618614Z","iopub.execute_input":"2022-04-21T07:55:54.618845Z","iopub.status.idle":"2022-04-21T07:55:54.635966Z","shell.execute_reply.started":"2022-04-21T07:55:54.618815Z","shell.execute_reply":"2022-04-21T07:55:54.635303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class HMDataset(torch.utils.data.Dataset):\n    def __init__(self, df):\n        self.df = df\n    def get_targetids(self, article_ids, week_relative):\n        max_week = np.max(week_relative)\n        targets = set()\n        \n        for i in range(len(article_ids)):\n            if week_relative[i] == max_week:\n                targets.add(article_ids[i])\n        \n        targets = list(targets)\n        np.random.shuffle(targets)\n        return targets[:config.MAX_TARGETS]\n    \n    def trim_target_time_ranges(self, article_ids, week_relative):\n        max_week = np.max(week_relative)\n        x_articles=[]\n        x_week = []\n        \n        for i in range(len(article_ids)):\n            if week_relative[i] != max_week:\n                x_articles.append(article_ids[i])\n                x_week.append(week_relative[i])\n        return (x_articles, x_week)\n        \n    \n    def pad_sequence(self, x, max_seqlen, padid):\n        x = x[-max_seqlen:]\n        seqdiff = max_seqlen - len(x)\n        if seqdiff == 0:\n            return x\n        x = ([padid]*seqdiff) + x\n        return x\n    \n    def encode_sequence(self, lst, article2id):\n        lst = [ article2id[x] for x in lst]\n        return lst\n    \n    def augment_weeks(self, num_weeks, week_relative, article_ids):\n        prv_week=-1\n        cnt=0\n        seqlen = len(week_relative)\n        num_train_weeks = np.random.choice( np.arange(4, num_weeks) )\n        \n        cur_seqlen = 0\n        for i in range( seqlen ):\n            if week_relative[i] != prv_week:\n                prv_week = week_relative[i]\n                cnt+=1\n            if cnt > num_train_weeks:\n                cur_seqlen = i\n                break\n        \n        week_relative = week_relative[:cur_seqlen]\n        article_ids = article_ids[:cur_seqlen]\n        \n        return (week_relative, article_ids)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        has_val = row.has_val\n        \n        train_data = row.train_val_split['train_data']\n        week_relative = train_data['week_relative']\n        article_ids = train_data['article_ids']\n        \n        if not has_val:\n            val_data = row.train_val_split['val_data']\n            week_relative += val_data['week_relative']\n            article_ids += val_data['article_ids']\n        \n        #Target ids\n        targets = self.get_targetids(article_ids, week_relative)\n        target_prodgroup_ids = [ article_info[articleid]['product_group_name'] for articleid in targets]\n        \n        \n        (article_ids, week_relative) = self.trim_target_time_ranges(article_ids, week_relative)\n        product_group_ids = [ article_info[articleid]['product_group_name'] for articleid in article_ids]\n        week_relative = list(np.clip(week_relative, 0, config.MAX_TIME_ID))\n        \n        #Encode Sequence\n        article_ids = self.encode_sequence(article_ids, article2id)\n        product_group_ids = self.encode_sequence(product_group_ids, prodgroup2id)\n        \n        targets  = self.encode_sequence(targets, article2id)\n        target_prodgroup_ids = self.encode_sequence(target_prodgroup_ids, prodgroup2id)\n        \n        seqlen = len(article_ids)\n        article_ids = self.pad_sequence(article_ids, config.MAX_SEQ_LEN, config.PAD_ARTICLE_ID)\n        product_group_ids = self.pad_sequence(product_group_ids, config.MAX_SEQ_LEN, config.PAD_PRODUCT_GROUP_ID)\n        week_relative = self.pad_sequence(week_relative, config.MAX_SEQ_LEN, config.PAD_TIME_ID)\n        \n        targets = self.pad_sequence(targets, config.MAX_TARGETS, config.PAD_ARTICLE_ID)\n        target_prodgroup_ids = self.pad_sequence(target_prodgroup_ids, config.MAX_TARGETS, config.PAD_PRODUCT_GROUP_ID)\n        \n        \n        #attn_mask - 1==> should ignore that position; 0 ==> donot ignore\n        attn_mask = torch.zeros(config.MAX_SEQ_LEN, dtype=torch.long).type(torch.bool)\n        attn_mask[:-seqlen]=True\n        \n        inputs = {\n            'article_ids': torch.tensor(article_ids, dtype=torch.long),\n            'product_group_ids': torch.tensor(product_group_ids, dtype=torch.long),\n            'week_relative': torch.tensor(week_relative, dtype=torch.long),\n            'seqlen': torch.tensor(seqlen, dtype=torch.long),\n            'attn_mask': attn_mask\n        }\n        \n        targets = torch.tensor(targets, dtype=torch.long)\n        target_prodgroup_ids = torch.tensor(target_prodgroup_ids, dtype=torch.long)\n        return inputs, targets\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.637961Z","iopub.execute_input":"2022-04-21T07:55:54.638241Z","iopub.status.idle":"2022-04-21T07:55:54.666344Z","shell.execute_reply.started":"2022-04-21T07:55:54.638212Z","shell.execute_reply":"2022-04-21T07:55:54.665299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(22)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.66857Z","iopub.execute_input":"2022-04-21T07:55:54.669213Z","iopub.status.idle":"2022-04-21T07:55:54.686292Z","shell.execute_reply.started":"2022-04-21T07:55:54.669176Z","shell.execute_reply":"2022-04-21T07:55:54.685322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ValDataset(torch.utils.data.Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.negsamples = np.random.choice( np.arange(len(article2id)), (len(self.df), config.NUM_NEGATIVE_SAMPLES) )\n        print(\"Numeb negsamples:\", self.negsamples.shape)\n        \n    def get_targetids(self, article_ids, week_relative):\n        max_week = np.max(week_relative)\n        targets = set()\n        \n        for i in range(len(article_ids)):\n            if week_relative[i] == max_week:\n                targets.add(article_ids[i])\n        \n        targets = list(targets)\n        np.random.shuffle(targets)\n        return targets[:config.MAX_TARGETS]\n    \n    def pad_sequence(self, x, max_seqlen, padid):\n        x = x[-max_seqlen:]\n        seqdiff = max_seqlen - len(x)\n        if seqdiff == 0:\n            return x\n        x = ([padid]*seqdiff) + x\n        return x\n    \n    def encode_sequence(self, lst, article2id):\n        lst = [ article2id[x] for x in lst]\n        return lst\n    \n    def get_train_sequence(self, train_data):\n        week_relative = train_data['week_relative']\n        article_ids = train_data['article_ids']\n        product_group_ids = [ article_info[articleid]['product_group_name'] for articleid in article_ids]\n        week_relative = list(np.clip(week_relative, 0, config.MAX_TIME_ID))\n        \n        #Encode Sequence\n        article_ids = self.encode_sequence(article_ids, article2id)\n        product_group_ids = self.encode_sequence(product_group_ids, prodgroup2id)\n        seqlen = len(article_ids)\n        \n        #Pad Sequences\n        article_ids = self.pad_sequence(article_ids, config.MAX_SEQ_LEN, config.PAD_ARTICLE_ID)\n        product_group_ids = self.pad_sequence(product_group_ids, config.MAX_SEQ_LEN, config.PAD_PRODUCT_GROUP_ID)\n        week_relative = self.pad_sequence(week_relative, config.MAX_SEQ_LEN, config.PAD_TIME_ID)\n        \n        #attn_mask - 1==> should ignore that position; 0 ==> donot ignore\n        attn_mask = torch.zeros(config.MAX_SEQ_LEN, dtype=torch.long).type(torch.bool)\n        attn_mask[:-seqlen]=True\n        \n        return {\n            'article_ids': torch.tensor(article_ids, dtype=torch.long),\n            'product_group_ids': torch.tensor(product_group_ids, dtype=torch.long),\n            'week_relative': torch.tensor(week_relative, dtype=torch.long),\n            'seqlen': torch.tensor(seqlen, dtype=torch.long),\n            'attn_mask': attn_mask\n        }\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        train_data = row.train_val_split['train_data']\n        val_data = row.train_val_split['val_data']\n        negsample = self.negsamples[idx]\n        \n        inputs = self.get_train_sequence(train_data)\n        targets = list(set(val_data['article_ids']))\n        targets  = self.encode_sequence(targets, article2id)\n        targets = self.pad_sequence(targets, config.MAX_TARGETS, config.PAD_ARTICLE_ID)\n        targets = torch.tensor(targets, dtype=torch.long)\n        \n        negsample = torch.tensor(negsample, dtype=torch.long)\n        \n        return inputs, targets, negsample\n    \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.688154Z","iopub.execute_input":"2022-04-21T07:55:54.688467Z","iopub.status.idle":"2022-04-21T07:55:54.709201Z","shell.execute_reply.started":"2022-04-21T07:55:54.68843Z","shell.execute_reply":"2022-04-21T07:55:54.708496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"128+32","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.710746Z","iopub.execute_input":"2022-04-21T07:55:54.711147Z","iopub.status.idle":"2022-04-21T07:55:54.727336Z","shell.execute_reply.started":"2022-04-21T07:55:54.71111Z","shell.execute_reply":"2022-04-21T07:55:54.726506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Embedding(nn.Module):\n    def __init__(self):\n        super(Embedding, self).__init__()\n        \n        self.article_embeddings = nn.Embedding(1+len(article2id), 128 ,padding_idx=config.PAD_ARTICLE_ID)\n        self.prodgroup_embeddings = nn.Embedding(1+len(prodgroup2id), 32 ,padding_idx=config.PAD_PRODUCT_GROUP_ID)\n        self.week_embeddings = nn.Embedding( 1 + config.PAD_TIME_ID, 160,padding_idx=config.PAD_TIME_ID)\n    \n    def get_article_embeddings(self, x_article):\n        x_article = self.article_embeddings(x_article)\n        return x_article\n    \n    def forward(self, x_week, x_article, x_prodgroup):\n        x_article = self.article_embeddings(x_article)\n        x_prodgroup = self.prodgroup_embeddings(x_prodgroup)\n        x_week = self.week_embeddings(x_week)\n        x = torch.cat([x_article, x_prodgroup], dim=-1)\n        x = x+x_week\n        return x\n    \nclass Encoder(nn.Module):\n    def __init__(self):\n        super(Encoder, self).__init__()\n        self.transformer_layer1 = nn.TransformerEncoderLayer(160, 4, dim_feedforward=512,\n                                                            dropout=0.15,\n                                                            batch_first=True,\n                                                            activation='gelu')\n        self.transformer_layer2 = nn.TransformerEncoderLayer(160, 4, dim_feedforward=512,\n                                                            dropout=0.15,\n                                                            batch_first=True,\n                                                            activation='gelu')\n        \n    def forward(self, x, attn_mask):\n        x = self.transformer_layer1(x, src_key_padding_mask=attn_mask)\n        x = self.transformer_layer2(x, src_key_padding_mask=attn_mask)\n        x = x.mean(dim=1)\n        return x\n    \nclass MLP(nn.Module):\n    def __init__(self):\n        super(MLP, self).__init__()\n        self.fc = nn.Sequential(\n            nn.Linear(160, 128),\n            nn.LeakyReLU(),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.1)\n        )\n        self.out = nn.Linear(128, 128)\n    def forward(self, x):\n        x = self.fc(x)\n        x = self.out(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.729512Z","iopub.execute_input":"2022-04-21T07:55:54.729868Z","iopub.status.idle":"2022-04-21T07:55:54.746758Z","shell.execute_reply.started":"2022-04-21T07:55:54.729833Z","shell.execute_reply":"2022-04-21T07:55:54.745828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.embedding_layer = Embedding()\n        self.encoder = Encoder()\n        self.mlp = MLP()\n    \n    def forward(self, x_week, x_article, x_prodgroup, attn_mask):\n        x = self.embedding_layer(x_week, x_article, x_prodgroup)\n        x = self.encoder(x, attn_mask)\n        x = self.mlp(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.747756Z","iopub.execute_input":"2022-04-21T07:55:54.748211Z","iopub.status.idle":"2022-04-21T07:55:54.761967Z","shell.execute_reply.started":"2022-04-21T07:55:54.748175Z","shell.execute_reply":"2022-04-21T07:55:54.760835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Negative Sampling","metadata":{}},{"cell_type":"code","source":"def get_train_negativesamples(articles):\n    bsize = articles.shape[0]\n    articles = set(articles.view(-1).tolist())\n    articles = list(articles.difference({config.PAD_ARTICLE_ID}))\n    \n    negsamples = np.random.choice(articles, (bsize, config.NUM_NEGATIVE_SAMPLES))\n    negsamples = torch.tensor(negsamples, dtype=torch.long, device=device)\n    return negsamples","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.763391Z","iopub.execute_input":"2022-04-21T07:55:54.763804Z","iopub.status.idle":"2022-04-21T07:55:54.773147Z","shell.execute_reply.started":"2022-04-21T07:55:54.763756Z","shell.execute_reply":"2022-04-21T07:55:54.772316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loss","metadata":{}},{"cell_type":"code","source":"def compute_loss(ylabels, hist_embedds, hpos, hneg):\n    loss = torch.tensor(0.0, device=device)\n    bsize=hist_embedds.shape[0]\n    dim=hist_embedds.shape[-1]\n    \n    pos_scores = hist_embedds.view(bsize, -1, dim) * hpos\n    neg_scores = hist_embedds.view(bsize, -1, dim) * hneg\n    \n    for i in range(config.MAX_TARGETS):\n        p = pos_scores[:, i, :].view(bsize, -1, dim)\n        score_diff = (p - neg_scores).sum(dim=-1)\n        cur_loss = -torch.log( score_diff.sigmoid() + 1e-9 )\n        cur_labels = ylabels[:, i]\n        \n        mask = (cur_labels != config.PAD_ARTICLE_ID)\n        cur_loss = cur_loss[mask, :]\n        loss += cur_loss.mean()\n    loss = loss/4\n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.774207Z","iopub.execute_input":"2022-04-21T07:55:54.774752Z","iopub.status.idle":"2022-04-21T07:55:54.786899Z","shell.execute_reply.started":"2022-04-21T07:55:54.774708Z","shell.execute_reply":"2022-04-21T07:55:54.786256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rank_metrics(ylabels, hist_embedds, hpos, hneg):\n    num_pos = 0\n    rank_5 = 0\n    rank_10 = 0\n    \n    bsize=hist_embedds.shape[0]\n    dim=hist_embedds.shape[-1]\n    \n    pos_scores = (hist_embedds.view(bsize, -1, dim) * hpos).sum(dim=-1)\n    neg_scores = (hist_embedds.view(bsize, -1, dim) * hneg).sum(dim=-1)\n    \n    for i in range(config.MAX_TARGETS):\n        p = pos_scores[:, i].view(bsize, -1)\n        score_diff = (p - neg_scores)\n        \n        cur_labels = ylabels[:, i]\n        mask = (cur_labels != config.PAD_ARTICLE_ID)\n        score_diff = (score_diff[mask, :] < 0).sum(dim=-1)\n        \n        num_pos += mask.sum().item()\n        rank_5 += (score_diff <= 5).sum().item()\n        rank_10 += (score_diff <= 10).sum().item()\n        \n    return rank_5, rank_10, num_pos","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.787838Z","iopub.execute_input":"2022-04-21T07:55:54.788436Z","iopub.status.idle":"2022-04-21T07:55:54.799051Z","shell.execute_reply.started":"2022-04-21T07:55:54.7884Z","shell.execute_reply":"2022-04-21T07:55:54.798371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train epoch","metadata":{}},{"cell_type":"code","source":"def train_epoch(train_dataloader, model, optimizer, schedular):\n    epoch_losses = []\n    model.train()\n    for it, (inputs, targets) in enumerate(train_dataloader):\n        batch_max_seqlen = inputs['seqlen'].max()\n\n        x_week  = inputs['week_relative'][:, -batch_max_seqlen:].to(device)\n        x_article = inputs['article_ids'][:, -batch_max_seqlen:].to(device)\n        x_prodgroup = inputs['product_group_ids'][:, -batch_max_seqlen:].to(device)\n        attn_mask = inputs['attn_mask'][:, -batch_max_seqlen:].to(device)\n        targets = targets.to(device)\n\n        negsamples = get_train_negativesamples(inputs['article_ids'])\n        hist_embedds = model(x_week, x_article, x_prodgroup, attn_mask)\n        pos_embedds = model.embedding_layer.get_article_embeddings(targets)\n        neg_embedds = model.embedding_layer.get_article_embeddings(negsamples)\n\n        \n        model.zero_grad(set_to_none=True)\n        loss = compute_loss(targets, hist_embedds, pos_embedds, neg_embedds)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 10)\n        loss.backward()\n        optimizer.step()\n        schedular.step()\n        \n        epoch_losses.append(loss.item())\n        if it%100 == 0 :\n            print(\"iteration:{} | loss:{:.4f}\".format(it, np.mean(epoch_losses)))\n    return np.mean(epoch_losses)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.800608Z","iopub.execute_input":"2022-04-21T07:55:54.801078Z","iopub.status.idle":"2022-04-21T07:55:54.811943Z","shell.execute_reply.started":"2022-04-21T07:55:54.801044Z","shell.execute_reply":"2022-04-21T07:55:54.811122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(val_dataloader, model):\n    model.eval()\n    losses = []\n    total_pos = 0.0\n    rank_5 = 0.0\n    rank_10 = 0.0\n    \n    for it, (inputs, targets, negsamples) in enumerate(val_dataloader):\n        batch_max_seqlen = inputs['seqlen'].max()\n\n        x_week  = inputs['week_relative'][:, -batch_max_seqlen:].to(device)\n        x_article = inputs['article_ids'][:, -batch_max_seqlen:].to(device)\n        x_prodgroup = inputs['product_group_ids'][:, -batch_max_seqlen:].to(device)\n        attn_mask = inputs['attn_mask'][:, -batch_max_seqlen:].to(device)\n        \n        targets = targets.to(device)\n        negsamples = negsamples.to(device)\n        with torch.no_grad():\n            hist_embedds = model(x_week, x_article, x_prodgroup, attn_mask)\n            pos_embedds = model.embedding_layer.get_article_embeddings(targets)\n            neg_embedds = model.embedding_layer.get_article_embeddings(negsamples)\n            \n            loss = compute_loss(targets, hist_embedds, pos_embedds, neg_embedds)\n            cur_rank_5, cur_rank_10, cur_total_pos = rank_metrics(targets, hist_embedds, pos_embedds, neg_embedds)\n            \n            total_pos += cur_total_pos\n            rank_5  += cur_rank_5\n            rank_10 += cur_rank_10\n            losses.append(loss.item())\n            \n    rank_5  = rank_5/total_pos\n    rank_10 = rank_10/total_pos\n    return np.mean(losses), rank_5, rank_10","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.813279Z","iopub.execute_input":"2022-04-21T07:55:54.813508Z","iopub.status.idle":"2022-04-21T07:55:54.827572Z","shell.execute_reply.started":"2022-04-21T07:55:54.813479Z","shell.execute_reply":"2022-04-21T07:55:54.82671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = HMDataset(transaction_df)\nval_dataset = ValDataset( transaction_df[transaction_df.has_val == True] )\n\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, shuffle=True, pin_memory=True, batch_size=config.BATCH_SIZE, drop_last=True)\nval_dataloader = torch.utils.data.DataLoader(val_dataset, shuffle=False, pin_memory=True, batch_size=config.BATCH_SIZE, drop_last=False)\n\nprint(\"Number of training iterations:\", len(train_dataloader))\nprint(\"Number of validation iterations:\", len(val_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:54.82898Z","iopub.execute_input":"2022-04-21T07:55:54.829691Z","iopub.status.idle":"2022-04-21T07:55:55.126123Z","shell.execute_reply.started":"2022-04-21T07:55:54.829653Z","shell.execute_reply":"2022-04-21T07:55:55.124895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model().to(device)\noptimizer=torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=0.001)\nscheduler = torch.optim.lr_scheduler.OneCycleLR(\n    optimizer,\n    1e-3,\n    epochs=config.NUM_EPOCHS,\n    steps_per_epoch=len(train_dataloader),\n    pct_start=0.05,\n    div_factor=100,\n    final_div_factor=10\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:55.127554Z","iopub.execute_input":"2022-04-21T07:55:55.127795Z","iopub.status.idle":"2022-04-21T07:55:55.276497Z","shell.execute_reply.started":"2022-04-21T07:55:55.127763Z","shell.execute_reply":"2022-04-21T07:55:55.275565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_loss = None\nbest_rank5 = None\nbest_rank10  =None\n\nfor e in range(config.NUM_EPOCHS):\n    epoch_loss = train_epoch(train_dataloader, model, optimizer, scheduler)\n    print(\"epoch: {} | train loss:{:.4f}\".format(e+1, epoch_loss))\n    \n    val_loss, rank_5, rank_10 = evaluate(val_dataloader, model)\n    if best_loss is None or val_loss < best_loss:\n        best_loss = val_loss\n        torch.save(model, \"model_best_loss.pt\")\n    if best_rank5 is None or rank_5 > best_rank5:\n        best_rank5 = rank_5\n        torch.save(model, \"model_best_rank_5.pt\")\n    \n    if best_rank10 is None or rank_10 > best_rank10:\n        best_rank10 = rank_10\n        torch.save(model, \"model_best_rank_10.pt\")\n    print(\"eval loss:{:.4f} | precision_5:{:.4f} | precision_10:{:.4f}\".format(val_loss, rank_5, rank_10))\n    print(\"best loss:{:.4f} | best rank5:{:.4f} | best rank10:{:.4f}\".format(best_loss, best_rank5, best_rank10))\n    \n    torch.save(model, \"model.pt\")\n    print(\"===\"*10)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T07:55:55.277673Z","iopub.execute_input":"2022-04-21T07:55:55.277896Z","iopub.status.idle":"2022-04-21T08:00:15.708111Z","shell.execute_reply.started":"2022-04-21T07:55:55.277857Z","shell.execute_reply":"2022-04-21T08:00:15.706732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}