{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **Installation . . .**","metadata":{"id":"iHcRKrNEbGNk"}},{"cell_type":"code","source":"import torch\nif not torch.cuda.is_available():\n    raise Exception(\"You should enable GPU runtime\")\n","metadata":{"id":"T5si03_5ELaD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","metadata":{"id":"VXyKg_GqEMkC"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Installing tensorboard and setting it up . . .**","metadata":{"id":"18g0kRL32ldP"}},{"cell_type":"markdown","source":"In this session, I wanted to use the original Tensorboard instead of using the TensorboardColab version. Doing this, for example, we are able to add images or graphs and not just scalars. Besides, we are able to load different experiments on the same graphics thus allowing us to compare them in the same plot.","metadata":{"id":"d31BpRdgtcPL"}},{"cell_type":"code","source":"# !pip install --upgrade tensorboard\n","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"pycharm":{"name":"#%%\n"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\n","metadata":{"id":"vI4CPvqJ2VWL","outputId":"4f25d984-c7f3-46b3-e761-6cb230276728"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nlogs_base_dir = \"runs\"\nos.makedirs(logs_base_dir, exist_ok=True)\n","metadata":{"id":"nMrWJHW82k4H"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter\n\ntb_fm = SummaryWriter(log_dir=f'{logs_base_dir}/{logs_base_dir}_FM/')\ntb_gcn = SummaryWriter(log_dir=f'{logs_base_dir}/{logs_base_dir}_GCN/')\ntb_gcn_attention = SummaryWriter(\n    log_dir=f'{logs_base_dir}/{logs_base_dir}_GCN_att/')\n","metadata":{"id":"avgB5dpoCkXE","pycharm":{"is_executing":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparing imports","metadata":{"id":"CLZh09ZZEh9Q"}},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\nfrom IPython import embed\nfrom sklearn.metrics import roc_auc_score\nimport pandas as pd\nimport numpy as np\nimport csv\nimport os\nimport scipy.sparse as sp\nfrom tqdm import tqdm, trange\n","metadata":{"id":"vw3DnPH5Ej4z"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Downloading data and loading it with pandas ...**","metadata":{"id":"TWJ7GgTNvEww"}},{"cell_type":"code","source":"# LOAD TRAINING DATA\ndata = pd.read_csv('./data/interactions.csv')\ndata\n","metadata":{"id":"Qlc-a62ihSoU","outputId":"e2b679c4-e8d2-45c8-fb82-b2d3020ade52"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unique value for the label is 1 (we will need to manually sample negative data)\ndata.nunique()\n","metadata":{"id":"2-EHrPNNiMDQ","outputId":"4f081b59-1c8d-41e7-885d-8a6a813b77a6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape\n","metadata":{"id":"XpmX0iGBibHS","outputId":"a719c6a0-1afb-413c-f9b3-9b113930b531"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.sort_values(by=['customer_id', 'timestamp'], ascending=[\n                        True, False], ignore_index=True)\ndata\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop_duplicates(\n    subset=['customer_id', 'article_id'], keep='first', inplace=True)\ndata\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[['customer_id', 'article_id']].copy()\ndata['label'] = 1\ndata\n","metadata":{"id":"7X3IorkGm0Yl"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, while we can observe for the training data many interactions for each user, we see below that for the testing (or validation) set we have just holded out one interaction for user, which will be used as ground-truth when evaluating the model outputing a ranking.","metadata":{"id":"8Ba0eE98yT4f"}},{"cell_type":"code","source":"data['idx'] = data.index\ndata\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = data.groupby('customer_id', as_index=False).first().set_index('idx')\ntest_data.index.name = None\ntest_data\ne = test_data.copy()\ne\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row_to_drop = test_data.index\ntest_data\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop(index=row_to_drop.values, columns=['idx'], inplace=True)\nd = data.copy()\ndata\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert 351760 - 338401 == 13359\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Preprocessing dataset ...**\n","metadata":{"id":"CqmuDc2083Sq"}},{"cell_type":"markdown","source":"We will first show how to preprocess data for some individual examples in `1. Understanding how to process data` section and finally we will construct a *Pytorch Dataset class* which will allow us to preprocess and handle the whole data in order to forward it to the model (it is done in `2. Building dataset and preparing data for the model` section).\n","metadata":{"id":"Y3f5Qm5yxLtD"}},{"cell_type":"markdown","source":"#### **1. Understanding how to process data...**","metadata":{"id":"NX5-Pvy-gOLp"}},{"cell_type":"code","source":"# userId,movieId,rating,timestamp\ndata = data.to_numpy()\ndata\n","metadata":{"id":"S9ztgsTXeDpa","outputId":"bb9fd2bd-79a3-44b8-ba19-efa072aa81ff"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items = data[:, :2].astype(int)\nitems\n","metadata":{"id":"03xEQjtgeOJh","outputId":"53cedcda-2d9e-4a6a-94bd-0c72cc3eb433"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.max(items, axis=0)[:2] + 1\n","metadata":{"id":"y5ZgSNuke_ye","outputId":"af368b4d-9b0a-4b07-b185-84fadedd7bca"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We need each node to have a unique id\nreindex_items = items.copy()\nreindex_items[:, 1] = reindex_items[:, 1] + 13359\nreindex_items\n","metadata":{"id":"UfvR59QefgU0","outputId":"887707f0-624f-4a77-9a05-886115ef16ee"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"field_dims = np.max(reindex_items, axis=0) + 1\nfield_dims\n","metadata":{"id":"UHQRN4GJft-a","outputId":"685d3e70-95e3-409d-a694-6b4dbfe18f36"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_adj_mx(dims, interactions):\n    train_mat = sp.dok_matrix((dims, dims), dtype=np.float32)\n    for x in tqdm(interactions, desc=\"BUILDING ADJACENCY MATRIX...\"):\n        train_mat[x[0], x[1]] = 1.0\n        train_mat[x[1], x[0]] = 1.0\n\n    return train_mat\n","metadata":{"id":"p8wVVlp4cOlY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mat = build_adj_mx(field_dims[-1], reindex_items.copy())\ntrain_mat\n","metadata":{"id":"ip9jrpD3gLGj","outputId":"457619ba-bd3d-46a4-9e6f-ed3f5cbf297b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check that we have (2*99057 = 198114) interactions...\n338401*2\n","metadata":{"id":"BxqwNJpxgpIA","outputId":"d9ba4128-6b2f-4487-c447-e549f4bce143"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### *Checking we have just positive data:*","metadata":{"id":"bjLXNJj-T2Cf"}},{"cell_type":"code","source":"targets = data[:, 2]\ntargets\n","metadata":{"id":"F2lKQf92YTJA","outputId":"db43dec2-aa52-406f-e090-be734d28924e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(targets)\n","metadata":{"id":"UnBLIE5en4zK","outputId":"ee9be814-c720-478c-f2b8-926f9ebd8d8c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### *Example on performing negative data for a training sample: (u, i, j)*","metadata":{"id":"iMxOy4uSmngm"}},{"cell_type":"code","source":"data = np.c_[(reindex_items, targets)].astype(int)\ndata\n","metadata":{"id":"P3ZGNi5iXoFW","outputId":"52b4dfb7-dd54-4e43-8de9-824f00901af8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"field_dims[:2]\n","metadata":{"id":"o7twzF5FYse6","outputId":"0c24de44-7672-4f64-ac4d-f876f7ba2015"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EXAMPLE interaction number 988 : user 6 - item 1470\nx = data[988]\nx\n","metadata":{"id":"uhY4DiE4ZkwS","outputId":"a7e5d8a0-9192-4ce3-e086-d396b8c081fe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg_triplet = np.array([0, 0, 0])\nneg_triplet[0] = x[0].copy()\nneg_triplet\n","metadata":{"id":"Q57mTXqLZm-8","outputId":"923d87fe-1b37-47c7-c348-246dbf306774"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example: We find item 1200 has no connection with user 6\nj = 1200\nneg_triplet[1] = j\nneg_triplet\n","metadata":{"id":"a2QLBkZWYHbd","outputId":"44231c39-c891-4c06-8e57-ed2bf985771d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### *Define metrics:*","metadata":{"id":"3whzmXbmUjMc"}},{"cell_type":"code","source":"import math\n\n\ndef getHitRatio(recommend_list, gt_item):\n    if gt_item in recommend_list:\n        return 1\n    else:\n        return 0\n\n\ndef getNDCG(recommend_list, gt_item):\n    idx = np.where(recommend_list == gt_item)[0]\n    if len(idx) > 0:\n        return math.log(2)/math.log(idx+2)\n    else:\n        return 0\n","metadata":{"id":"uMhyY-hVUjWX"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### *Build test dataset for evaluation*","metadata":{"id":"A2pw4pMTmr_d"}},{"cell_type":"code","source":"test_data\n","metadata":{"id":"ELX6l9N-eifC","outputId":"6feb18f6-53e4-40d9-edf7-a44ed820dcb5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = test_data.to_numpy()\ntest_data.shape[0]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Take number of users and items from reindex items from train set\nusers, items = np.max(reindex_items, axis=0)[:2] + 1  # [ 943, 1682])\nprint(users)\nprint(items)\n","metadata":{"id":"dzfx621OiaRL","outputId":"2946283e-473b-432a-c34e-5fe5db485a7d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reindex test items and substract 1\npairs_test = test_data[:, :2].astype(int)\npairs_test[:, 1] = pairs_test[:, 1] + users\npairs_test\n","metadata":{"id":"FQ9T2YGvcn2w","outputId":"cd64731b-5101-4dbd-e623-d91e954c5a23"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert 1219 + 13359 == 14578\n","metadata":{"id":"kptPm6E4ozIZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pair = pairs_test[0]\npair\n","metadata":{"id":"Ad4BNmcCiN7v","outputId":"d141e672-a0ee-4d41-8036-61daceb9765f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GENERATE TEST SET WITH NEGATIVE EXAMPLES TO EVALUATE\n# number users (943), number items (2625)\nmax_users, max_items = field_dims[:2]\nnegatives = []\nfor t in range(10):\n    j = np.random.randint(max_users, max_items)\n    while (pair[0], j) in train_mat or j == pair[1]:\n        j = np.random.randint(max_users, max_items)\n    negatives.append(j)\nnegatives\n","metadata":{"id":"ddiimRXgkCLW","outputId":"6ded9863-14a9-480a-a412-32a10596a7c6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"single_user_test_set = np.vstack([pair, ] * (len(negatives)+1))\nsingle_user_test_set\n","metadata":{"id":"yNs5YdPhxJeL","outputId":"e9d1bd7e-9b5b-4f80-e6b8-53ceb725ff48"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"single_user_test_set[:, 1][1:] = negatives\nsingle_user_test_set\n","metadata":{"id":"wJoln4vNwxyK","outputId":"44c371dd-443f-4f1a-bc4a-95cf6fb9cb67"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **2. Building dataset and preparing data for the model ...**","metadata":{"id":"Mz5TAC52gVPz"}},{"cell_type":"code","source":"d.to_numpy()[:, 2]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"e\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# @title\nimport numpy as np\nimport pandas as pd\nimport torch.utils.data\n\n\nclass HandMDataset(torch.utils.data.Dataset):\n    \"\"\"\n    MovieLens 100k Dataset\n\n    Data preparation\n        treat samples with a rating less than 3 as negative samples\n\n    :param dataset_path: MovieLens dataset path\n\n    \"\"\"\n\n    def __init__(self, train_data, test_data, num_negatives_train=4, num_negatives_test=100):\n\n        colnames = [\"user_id\", 'item_id', 'label']\n        data = train_data.to_numpy()\n        test_data = test_data.to_numpy()\n\n        # TAKE items, targets and test_items\n        self.targets = data[:, 2]\n        self.items = self.preprocess_items(data)\n\n        # Save dimensions of max users and items and build training matrix\n        self.field_dims = np.max(self.items, axis=0) + 1  # ([ 943, 2625])\n        print(self.field_dims)\n        self.train_mat = self.__build_adj_mx__(self.field_dims[-1], self.items.copy())\n\n        # Generate train interactions with 4 negative samples for each positive\n        self.negative_sampling(num_negatives=num_negatives_train)\n\n        # Build test set by passing as input the test item interactions\n        self.test_set = self.build_test_set(self.preprocess_items(test_data), num_neg_samples_test=num_negatives_test)\n\n    def __len__(self):\n        return self.targets.shape[0]\n\n    def __getitem__(self, index):\n        return self.interactions[index]\n\n    def preprocess_items(self, data, users=13359):\n        reindexed_items = data[:, :2].astype(\n            np.int)  # -1 because ID begins from 1\n        # users, items = np.max(reindexed_items, axis=0)[:2] + 1 # [ 943, 1682])\n        # Reindex items (we need to have [users + items] nodes with unique idx)\n        reindexed_items[:, 1] = reindexed_items[:, 1] + users\n\n        return reindexed_items\n    \n    def __build_adj_mx__(self, dims, interactions):\n        train_mat = sp.dok_matrix((dims, dims), dtype=np.float32)\n        for x in tqdm(interactions, desc=\"BUILDING ADJACENCY MATRIX...\"):\n            train_mat[x[0], x[1]] = 1.0\n            train_mat[x[1], x[0]] = 1.0\n\n        return train_mat\n\n    def negative_sampling(self, num_negatives=4):\n        self.interactions = []\n        data = np.c_[(self.items, self.targets)].astype(int)\n        # number users (943), number items (2625)\n        max_users, max_items = self.field_dims[:2]\n\n        # x are triplets (u, i , 1)\n        for x in tqdm(data, desc=\"Performing negative sampling on test data...\"):\n            # Append positive interaction\n            self.interactions.append(x)\n            # Copy user and maintain last position to 0. Now we will need to update neg_triplet[1] with j\n            neg_triplet = np.vstack([x, ] * (num_negatives))\n            neg_triplet[:, 2] = np.zeros(num_negatives)\n\n            # Generate num_negatives negative interactions\n            for idx in range(num_negatives):\n                j = np.random.randint(max_users, max_items)\n                # IDEA: Loop to exclude true interactions (set to 1 in adj_train) user - item\n                while (x[0], j) in self.train_mat:\n                    j = np.random.randint(max_users, max_items)\n                neg_triplet[:, 1][idx] = j\n            self.interactions.append(neg_triplet.copy())\n\n        self.interactions = np.vstack(self.interactions)\n\n    def build_test_set(self, gt_test_interactions, num_neg_samples_test=99):\n        # number users (943), number items (2625)\n        max_users, max_items = self.field_dims[:2]\n        test_set = []\n        for pair in tqdm(gt_test_interactions, desc=\"BUILDING TEST SET...\"):\n            negatives = []\n            for t in range(num_neg_samples_test):\n                j = np.random.randint(max_users, max_items)\n                while (pair[0], j) in self.train_mat or j == pair[1]:\n                    j = np.random.randint(max_users, max_items)\n                negatives.append(j)\n            # APPEND TEST SETS FOR SINGLE USER\n            single_user_test_set = np.vstack([pair, ] * (len(negatives)+1))\n            single_user_test_set[:, 1][1:] = negatives\n            test_set.append(single_user_test_set.copy())\n        return test_set\n","metadata":{"id":"H7IkNtA2LeR6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_dataset = HandMDataset(d, e, num_negatives_train=4, num_negatives_test=99)","metadata":{"id":"Okf_CKu-hQH6","outputId":"7c11cc77-ef61-46ba-cecb-71cda263e0f0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 90570 interactions with pairs of index that have interacted + 4*90570 negative\nfull_dataset.interactions","metadata":{"id":"1kVPuJr0htZi","outputId":"2b80acd9-5e73-4d39-83bf-d61c06e32903"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_dataset.interactions[:20]","metadata":{"id":"yDarK-6QT7BT","outputId":"b8452d44-021b-4abb-b4df-ae89f8ffb998"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We had 99057 interactions in training_matrix --> now we have 99057 positive plus 4*99057 negative\nassert 5*338401 == full_dataset.interactions.shape[0]","metadata":{"id":"u1rrSiUnoEGu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For test set, we keep the size (one interaction per user) but we append 99 negative samples for evaluation\nprint(len(full_dataset.test_set))\n","metadata":{"id":"EjTmCleF8HR5","outputId":"54933f52-19b7-4cba-8673-60fee8255c57"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(full_dataset.test_set[0])  # --> [gt_pair + 99_neg_samples]\n","metadata":{"id":"mLB3P3tnywKg","outputId":"c714c4da-545a-4fb8-ebe1-4a1dd6138a63"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_dataset.test_set[0]\n","metadata":{"id":"wmDTn5mzUIKG","outputId":"dd271357-9f24-4442-ad3e-8794e455836a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sampling 4 negative samples for each positive, will also work as a type of normalization.","metadata":{"id":"RXEAH46wppTY"}},{"cell_type":"code","source":"data_loader = DataLoader(full_dataset, batch_size=256, shuffle=True, num_workers=0)","metadata":{"id":"VROSRVsH9NNb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (interactions) in enumerate(data_loader):\n    if i == 0:\n        print(interactions.shape)\n    else:\n        break\n","metadata":{"id":"vTvD_ftK9NUe","outputId":"22541592-81ad-44f3-bf48-9b8e6cf8b104"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Building Factorization Machines model**","metadata":{"id":"xNAKSGghsm7_"}},{"cell_type":"markdown","source":"\nOur training matrix is now even sparser: Of all 237,746,250 values (90,570*2,625), only 181,140 are non-zero (90,570*2). In other words, the matrix is 99.92% sparse. Storing this as a dense matrix would be a massive waste of both storage and computing power!\nTo avoid this, let’s use a scipy.lil_matrix sparse matrix for samples and a numpy array for labels.\n","metadata":{"id":"rdgA3E4Js4mx"}},{"cell_type":"markdown","source":"\n<div>\n<center><img src=\"https://d2908q01vomqb2.cloudfront.net/f1f836cb4ea6efb2a0b1b99f41ad8b103eff4b59/2019/04/03/sagemaker-factorization-1.gif\" width=\"400\"/></center>\n</div>","metadata":{"id":"GSlwmfoK0mA-"}},{"cell_type":"markdown","source":"##### **LAYERS:** Linear and FM part of the equation","metadata":{"id":"cMmrziB8rvya"}},{"cell_type":"code","source":"# EMBEDDING PYTORCH: https://pytorch.org/docs/stable/nn.html?highlight=embedding#torch.nn.Embedding\n","metadata":{"id":"eSFpvgYTu4Xg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Linear part of the equation\nclass FeaturesLinear(torch.nn.Module):\n\n    def __init__(self, field_dims, output_dim=1):\n        super().__init__()\n\n        self.fc = torch.nn.Embedding(field_dims, output_dim)\n        self.bias = torch.nn.Parameter(torch.zeros((output_dim,)))\n\n    def forward(self, x):\n        \"\"\"\n        :param x: Long tensor of size ``(batch_size, num_fields)``\n        \"\"\"\n        # self.fc(x).shape --> [batch_size, num_fields, 1]\n        # torch.sum(self.fc(x), dim=1).shape --> ([batch_size, 1])\n        return torch.sum(self.fc(x), dim=1) + self.bias\n","metadata":{"id":"0cyYo9pXlSEc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FM part of the equation\nclass FM_operation(torch.nn.Module):\n\n    def __init__(self, reduce_sum=True):\n        super().__init__()\n        self.reduce_sum = reduce_sum\n\n    def forward(self, x):\n        \"\"\"\n        :param x: Float tensor of size ``(batch_size, num_fields, embed_dim)``\n        \"\"\"\n        square_of_sum = torch.sum(x, dim=1) ** 2\n        sum_of_square = torch.sum(x ** 2, dim=1)\n        ix = square_of_sum - sum_of_square\n        if self.reduce_sum:\n            ix = torch.sum(ix, dim=1, keepdim=True)\n        return 0.5 * ix\n","metadata":{"id":"C7ibkJzxyCdt"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### MODEL","metadata":{"id":"Bw-jiLKCsCDk"}},{"cell_type":"code","source":"from torch_geometric.nn import GCNConv\n\n\nclass FactorizationMachineModel(torch.nn.Module):\n    \"\"\"\n    A pytorch implementation of Factorization Machine.\n\n    Reference:\n        S Rendle, Factorization Machines, 2010.\n    \"\"\"\n\n    def __init__(self, field_dims, embed_dim):\n        super().__init__()\n        # field_dims == total of nodes (sum users + context)\n        # self.linear = torch.nn.Linear(field_dims, 1, bias=True)\n        self.linear = FeaturesLinear(field_dims)\n        self.embedding = torch.nn.Embedding(\n            field_dims, embed_dim, sparse=False)\n        self.fm = FM_operation(reduce_sum=True)\n\n        torch.nn.init.xavier_uniform_(self.embedding.weight.data)\n\n    def forward(self, interaction_pairs):\n        \"\"\"\n        :param interaction_pairs: Long tensor of size ``(batch_size, num_fields)``\n        \"\"\"\n        out = self.linear(interaction_pairs) + \\\n            self.fm(self.embedding(interaction_pairs))\n\n        return out.squeeze(1)\n\n    def predict(self, interactions, device):\n        # return the score, inputs are numpy arrays, outputs are tensors\n        test_interactions = torch.from_numpy(\n            interactions).to(dtype=torch.long, device=device)\n        output_scores = self.forward(test_interactions)\n        return output_scores\n","metadata":{"id":"agCg8QZaxyAt"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Workflow for FM with usual embeddings ...**","metadata":{"id":"MpTBEpD3C-_P"}},{"cell_type":"markdown","source":"#### **Train**","metadata":{"id":"O7jtY86it__d"}},{"cell_type":"code","source":"from statistics import mean\n\n\ndef train_one_epoch(model, optimizer, data_loader, criterion, device, log_interval=100):\n    model.train()\n    total_loss = []\n\n    for i, (interactions) in enumerate(data_loader):\n        interactions = interactions.to(device)\n        targets = interactions[:, 2]\n        predictions = model(interactions[:, :2])\n\n        loss = criterion(predictions, targets.float())\n        model.zero_grad()\n        loss.backward()\n        optimizer.step()\n        total_loss.append(loss.item())\n\n    return mean(total_loss)\n","metadata":{"id":"AUDsUfdUDCkJ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Evaluation**","metadata":{"id":"FGbK80gNuEhT"}},{"cell_type":"markdown","source":"##### **Understanding evaluation ...**","metadata":{"id":"AmtwbJ9HzuBQ"}},{"cell_type":"code","source":"len(full_dataset.test_set)\n","metadata":{"id":"NCCH3ay7vUsm","outputId":"a705fdef-2bb0-4aeb-d744-513d491b1525"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_test = full_dataset.test_set[0]\nuser_test.shape\n","metadata":{"id":"dgwLEHv_vjRi","outputId":"e5c0940a-dd7f-46f6-a303-2eb65d6a3bff"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_test\n","metadata":{"id":"1_4TUpWClpDo","outputId":"099dea96-5271-438c-abce-a905e22129cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt_pair = user_test[0]\nneg_items = user_test[1:]\nprint(f'gt_pair: {gt_pair}')\nprint(f'lenght neg_items: {len(neg_items)}')\n","metadata":{"id":"XYS-E499vpmZ","outputId":"e6e905f9-cd93-46ea-a9af-7892a6deb868"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DEFINE GT_ITEM\ngt_item = user_test[0][1]\ngt_item\n","metadata":{"id":"FoM-OgFB3BeG","outputId":"d15684ec-aedf-43ed-bc1b-ef47b9ce537e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining dummy model with 8 embedding dimensions\ndummy_model = FactorizationMachineModel(\n    full_dataset.field_dims[-1], 8).to(device)\nout = dummy_model.predict(user_test, device)\nout.shape\n","metadata":{"id":"7xszUI7bzarC","outputId":"3d890a48-8c34-4153-9a0d-ed4f27e240b1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print first 10 predictions, where 1st one is the one for the GT\nout[:10]\n","metadata":{"id":"0h28_id_zh5_","outputId":"d90159e4-7298-4f37-c74a-064c712690cf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"values, indices = torch.topk(out, 10)\nprint(values)\nprint(indices.cpu().detach().numpy())\n","metadata":{"id":"LUpRWF6F1gyd","outputId":"3c783fc3-4cf4-4b9e-fc14-bdcafa9904e1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_test[0]\n","metadata":{"id":"CBpBD5Rp2pyu","outputId":"cc835e1b-e036-42bd-8ce6-7cead82c4786"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# RANKING LIST TO RECOMMEND\nrecommend_list = user_test[indices.cpu().detach().numpy()][:, 1]\nrecommend_list\n","metadata":{"id":"oWiJr5OA1qjT","outputId":"8e4287d3-6f5d-4ea9-b507-0e0e63851bbb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt_item in recommend_list\n","metadata":{"id":"-LZ0Iz_63gzF","outputId":"d9c716b3-55ec-49e7-ff72-6f1a1a9c16c6"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### **Defining test function...**","metadata":{"id":"vlva2UiB6LkI"}},{"cell_type":"code","source":"def test(model, full_dataset, device, topk=10):\n    # Test the HR and NDCG for the model @topK\n    model.eval()\n\n    HR, NDCG = [], []\n\n    for user_test in full_dataset.test_set:\n        gt_item = user_test[0][1]\n\n        predictions = model.predict(user_test, device)\n        _, indices = torch.topk(predictions, topk)\n        recommend_list = user_test[indices.cpu().detach().numpy()][:, 1]\n\n        HR.append(getHitRatio(recommend_list, gt_item))\n        NDCG.append(getNDCG(recommend_list, gt_item))\n    return mean(HR), mean(NDCG)\n","metadata":{"id":"T43Y3aM-RM-U"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Model, loss and optimizer definition**","metadata":{"id":"qWkS4HJFuLyL"}},{"cell_type":"code","source":"model = FactorizationMachineModel(full_dataset.field_dims[-1], 32).to(device)\n","metadata":{"id":"byFW63h-Dkel"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss(reduction='mean')\noptimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n","metadata":{"id":"-MSFVW-yDCrJ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Random evaluation**","metadata":{"id":"6gYl3_7Iph14"}},{"cell_type":"code","source":"topk = 10\n\n# Check Init performance\nhr, ndcg = test(model, full_dataset, device, topk=topk)\nprint(\"initial HR: \", hr)\nprint(\"initial NDCG: \", ndcg)\n","metadata":{"id":"UKK4U2LPPlR0","outputId":"ed7f8a00-51bd-4cfb-c483-6cc68c04bcd7"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### **Start training the model**","metadata":{"id":"LHbKyX8vpqOa"}},{"cell_type":"code","source":"# DO EPOCHS NOW\ntb = True\ntopk = 10\nfor epoch_i in range(20):\n    # data_loader.dataset.negative_sampling()\n    train_loss = train_one_epoch(\n        model, optimizer, data_loader, criterion, device)\n    hr, ndcg = test(model, full_dataset, device, topk=topk)\n\n    print('\\n')\n\n    print(f'epoch {epoch_i}:')\n    print(\n        f'training loss = {train_loss:.4f} | Eval: HR@{topk} = {hr:.4f}, NDCG@{topk} = {ndcg:.4f} ')\n    print('\\n')\n    if tb:\n        tb_fm.add_scalar('train/loss', train_loss, epoch_i)\n        tb_fm.add_scalar('eval/HR@{topk}', hr, epoch_i)\n        tb_fm.add_scalar('eval/NDCG@{topk}', ndcg, epoch_i)\n","metadata":{"id":"4gs86M3iUXFM","outputId":"672af415-8283-4aaf-df63-e5c4ea2c6b70"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **VISUALIZING RESULTS**\n\nOnce we have trained both models (*fm with usual embbedding layers* vs *fm with embeddings from gcn*), we can observe both metrics and loss in the same graphic in order to compare:","metadata":{"id":"tMfW67UKWkfT"}},{"cell_type":"code","source":"%tensorboard --logdir runs","metadata":{"id":"MbSDTbdNdH3o"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OUR MODEL ","metadata":{}},{"cell_type":"code","source":"from torch_geometric.nn import GCNConv\nfrom scipy.sparse import identity\nfrom torch_geometric.utils import dense_to_sparse, from_scipy_sparse_matrix\n\n\nclass GCNModel(torch.nn.Module):\n    def __init__(self, field_dims, embed_dim, A):\n        super().__init__()\n        self.features = torch.Tensor(np.identity(A.shape[0])).to(device)\n        self.gcn = GCNConv(field_dims, embed_dim)\n        self.A = A\n        self.edge_index_t, self.edge_weights = from_scipy_sparse_matrix(self.A)\n\n        self.edge_index_t = self.edge_index_t.to(device)\n\n    def forward(self, x):\n        # x, edge_index = x.x, x.edge_index\n        gcn = self.gcn(self.features, self.edge_index_t)[x]\n        return gcn\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FMGCNModel(torch.nn.Module):\n    def __init__(self, field_dims, embed_dim, A):\n        super().__init__()\n        # field_dims == total of nodes (sum users + context)\n        # self.linear = torch.nn.Linear(field_dims, 1, bias=True)\n        self.linear = FeaturesLinear(field_dims)\n        # self.embedding = torch.nn.Embedding(field_dims, embed_dim, sparse=False)\n        self.embedding = GCNModel(field_dims, embed_dim, A)\n        self.fm = FM_operation(reduce_sum=True)\n\n        # torch.nn.init.xavier_uniform_(self.embedding.)\n\n    def forward(self, interaction_pairs):\n        \"\"\"\n        :param interaction_pairs: Long tensor of size ``(batch_size, num_fields)``\n        \"\"\"\n        out = self.linear(interaction_pairs) + \\\n            self.fm(self.embedding(interaction_pairs))\n\n        return out.squeeze(1)\n\n    def predict(self, interactions, device):\n        # return the score, inputs are numpy arrays, outputs are tensors\n        test_interactions = torch.from_numpy(\n            interactions).to(dtype=torch.long, device=device)\n        output_scores = self.forward(test_interactions)\n        return output_scores\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining dummy model with 8 embedding dimensions\ndummy_model = FMGCNModel(full_dataset.field_dims[-1], 8, full_dataset.train_mat).to(device)\nout = dummy_model.predict(user_test, device)\nout.shape\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = FMGCNModel(full_dataset.field_dims[-1], 32, full_dataset.train_mat).to(device)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss(reduction='mean')\noptimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DO EPOCHS NOW\ntb = True\ntopk = 10\nfor epoch_i in range(20):\n    # data_loader.dataset.negative_sampling()\n    train_loss = train_one_epoch(\n        model, optimizer, data_loader, criterion, device)\n    hr, ndcg = test(model, full_dataset, device, topk=topk)\n\n    print('\\n')\n\n    print(f'epoch {epoch_i}:')\n    print(\n        f'training loss = {train_loss:.4f} | Eval: HR@{topk} = {hr:.4f}, NDCG@{topk} = {ndcg:.4f} ')\n    print('\\n')\n    if tb:\n        tb_gcn.add_scalar('train/loss', train_loss, epoch_i)\n        tb_gcn.add_scalar('eval/HR@{topk}', hr, epoch_i)\n        tb_gcn.add_scalar('eval/NDCG@{topk}', ndcg, epoch_i)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch_geometric.nn import GATConv\n\nclass GCNAttModel(torch.nn.Module):\n    def __init__(self, field_dims, embed_dim, A):\n        super().__init__()\n        self.features = torch.Tensor(np.identity(A.shape[0])).to(device)\n        self.gcn = GATConv(int(field_dims), embed_dim)         \n        self.A = A\n        self.edge_index_t, self.edge_weights = from_scipy_sparse_matrix(self.A)\n\n        self.edge_index_t = self.edge_index_t.to(device)\n\n    def forward(self, x):\n        # x, edge_index = x.x, x.edge_index\n        gcn = self.gcn(self.features, self.edge_index_t)[x]\n        return gcn\n    \n\nclass FMGCNAttModel(torch.nn.Module):\n    def __init__(self, field_dims, embed_dim, A):\n        super().__init__()\n        # field_dims == total of nodes (sum users + context)\n        # self.linear = torch.nn.Linear(field_dims, 1, bias=True)\n        self.linear = FeaturesLinear(field_dims)\n        self.embedding = GCNAttModel(field_dims, embed_dim, A)\n        # self.embedding = GCNModel(field_dims, embed_dim, A)\n        self.fm = FM_operation(reduce_sum=True)\n\n        # torch.nn.init.xavier_uniform_(self.embedding.)\n\n    def forward(self, interaction_pairs):\n        \"\"\"\n        :param interaction_pairs: Long tensor of size ``(batch_size, num_fields)``\n        \"\"\"\n        out = self.linear(interaction_pairs) + \\\n            self.fm(self.embedding(interaction_pairs))\n\n        return out.squeeze(1)\n\n    def predict(self, interactions, device):\n        # return the score, inputs are numpy arrays, outputs are tensors\n        test_interactions = torch.from_numpy(\n            interactions).to(dtype=torch.long, device=device)\n        output_scores = self.forward(test_interactions)\n        return output_scores\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\nimport gc\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(full_dataset.field_dims[-1])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining dummy model with 8 embedding dimensions\ndummy_model = FMGCNAttModel(full_dataset.field_dims[-1], 8, full_dataset.train_mat).to(device)\nout = dummy_model.predict(user_test, device)\nout.shape\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = FMGCNAttModel(full_dataset.field_dims[-1], 32, full_dataset.train_mat).to(device)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss(reduction='mean')\noptimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DO EPOCHS NOW\ntb = True\ntopk = 10\nfor epoch_i in range(20):\n    # data_loader.dataset.negative_sampling()\n    train_loss = train_one_epoch(\n        model, optimizer, data_loader, criterion, device)\n    hr, ndcg = test(model, full_dataset, device, topk=topk)\n\n    print('\\n')\n\n    print(f'epoch {epoch_i}:')\n    print(\n        f'training loss = {train_loss:.4f} | Eval: HR@{topk} = {hr:.4f}, NDCG@{topk} = {ndcg:.4f} ')\n    print('\\n')\n    if tb:\n        tb_gcn_attention.add_scalar('train/loss', train_loss, epoch_i)\n        tb_gcn_attention.add_scalar('eval/HR@{topk}', hr, epoch_i)\n        tb_gcn_attention.add_scalar('eval/NDCG@{topk}', ndcg, epoch_i)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%tensorboard --logdir runs","metadata":{},"execution_count":null,"outputs":[]}]}