{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pickle5","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-14T18:54:40.404279Z","iopub.execute_input":"2023-03-14T18:54:40.404680Z","iopub.status.idle":"2023-03-14T18:54:50.242759Z","shell.execute_reply.started":"2023-03-14T18:54:40.404647Z","shell.execute_reply":"2023-03-14T18:54:50.241454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:55:44.770819Z","iopub.execute_input":"2023-03-14T18:55:44.771203Z","iopub.status.idle":"2023-03-14T18:55:56.219991Z","shell.execute_reply.started":"2023-03-14T18:55:44.771160Z","shell.execute_reply":"2023-03-14T18:55:56.218751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install merlin-dataloader==0.0.2","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-14T18:54:50.245459Z","iopub.execute_input":"2023-03-14T18:54:50.246291Z","iopub.status.idle":"2023-03-14T18:55:00.181146Z","shell.execute_reply.started":"2023-03-14T18:54:50.246240Z","shell.execute_reply":"2023-03-14T18:55:00.179924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nfrom collections import defaultdict\nimport pickle5 as p \nfrom tqdm.autonotebook import tqdm\nimport os\nfrom merlin.loader.torch import Loader \nfrom merlin.io import Dataset","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:55:56.222296Z","iopub.execute_input":"2023-03-14T18:55:56.222992Z","iopub.status.idle":"2023-03-14T18:56:00.466223Z","shell.execute_reply.started":"2023-03-14T18:55:56.222947Z","shell.execute_reply":"2023-03-14T18:56:00.465245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/mf-covisitation')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:59:04.504104Z","iopub.execute_input":"2023-03-14T18:59:04.504874Z","iopub.status.idle":"2023-03-14T18:59:04.510638Z","shell.execute_reply.started":"2023-03-14T18:59:04.504835Z","shell.execute_reply":"2023-03-14T18:59:04.509445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n <h1  id=\"1\" style=\"color:white;\n           padding:10px;\n           display:fill;\n           border-radius:10px;\n           background-color:#5642C5;\n           font-size:200%;\n           font-family:Verdana;\n           letter-spacing:0.5px\">  1) Matrix Factorization Features for Local </h1>\n","metadata":{}},{"cell_type":"code","source":"RUN_FOR = 'local'\ntrain = pl.read_parquet('/kaggle/input/convert-ecommerce-data-to-otto-format/train.parquet')\nval_A = pl.read_parquet('/kaggle/input/otto-surrogate-feature-generation/data_splits/val_A.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:53:36.529697Z","iopub.status.idle":"2023-03-14T18:53:36.530120Z","shell.execute_reply.started":"2023-03-14T18:53:36.529878Z","shell.execute_reply":"2023-03-14T18:53:36.529902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to create `aid-aid` pairs to train our matrix factorization model.\n\nGrabbing the pairs both from the train and test set.","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain_pairs = (pl.concat([train, val_A])\n    .sort(['session','timestamp'])\n    .groupby('session').agg([\n        pl.col('aid'),\n        pl.col('aid').shift(-1).alias('aid_next')\n    ])\n    .explode(['aid', 'aid_next'])\n    .drop_nulls()\n)[['aid', 'aid_next']]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:25:46.384763Z","iopub.execute_input":"2023-03-14T17:25:46.385457Z","iopub.status.idle":"2023-03-14T17:25:47.942367Z","shell.execute_reply.started":"2023-03-14T17:25:46.385421Z","shell.execute_reply":"2023-03-14T17:25:47.941353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:25:59.918157Z","iopub.execute_input":"2023-03-14T17:25:59.918532Z","iopub.status.idle":"2023-03-14T17:25:59.929169Z","shell.execute_reply.started":"2023-03-14T17:25:59.918501Z","shell.execute_reply":"2023-03-14T17:25:59.928040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:28:06.458991Z","iopub.execute_input":"2023-03-14T17:28:06.459596Z","iopub.status.idle":"2023-03-14T17:28:06.466125Z","shell.execute_reply.started":"2023-03-14T17:28:06.459560Z","shell.execute_reply":"2023-03-14T17:28:06.465140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Retrieving the cardinality of our aids -- we will need this to create the embedding layer.","metadata":{}},{"cell_type":"code","source":"cardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())\ncardinality_aids","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:26:15.802656Z","iopub.execute_input":"2023-03-14T17:26:15.803041Z","iopub.status.idle":"2023-03-14T17:26:15.811886Z","shell.execute_reply.started":"2023-03-14T17:26:15.803005Z","shell.execute_reply":"2023-03-14T17:26:15.810894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using Merlin Dataloader to load the data","metadata":{}},{"cell_type":"code","source":"train_pairs[:-800_000].to_pandas().to_parquet('train_pairs.parquet')\ntrain_pairs[-800_000:].to_pandas().to_parquet('valid_pairs.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:28:17.081304Z","iopub.execute_input":"2023-03-14T17:28:17.081683Z","iopub.status.idle":"2023-03-14T17:28:17.323359Z","shell.execute_reply.started":"2023-03-14T17:28:17.081652Z","shell.execute_reply":"2023-03-14T17:28:17.322288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = Dataset('train_pairs.parquet')\ntrain_dl_merlin = Loader(train_ds, 65536, True)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:28:19.554514Z","iopub.execute_input":"2023-03-14T17:28:19.555184Z","iopub.status.idle":"2023-03-14T17:28:31.275228Z","shell.execute_reply.started":"2023-03-14T17:28:19.555147Z","shell.execute_reply":"2023-03-14T17:28:31.274197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Testing how long it takes to iterate over the dataset using dataloader:","metadata":{}},{"cell_type":"code","source":"%%time\n\nfor batch, _ in train_dl_merlin:\n    aid1, aid2 = batch['aid'], batch['aid_next']","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:28:44.841296Z","iopub.execute_input":"2023-03-14T17:28:44.841648Z","iopub.status.idle":"2023-03-14T17:28:46.170772Z","shell.execute_reply.started":"2023-03-14T17:28:44.841613Z","shell.execute_reply":"2023-03-14T17:28:46.169523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Original MF model without modifications","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.optim import SparseAdam\n\nclass MatrixFactorization(nn.Module):\n    def __init__(self, n_aids, n_factors):\n        super().__init__()\n        self.aid_factors = nn.Embedding(n_aids, n_factors, sparse=True)\n        \n    def forward(self, aid1, aid2):\n        aid1 = self.aid_factors(aid1)\n        aid2 = self.aid_factors(aid2)\n        \n        return (aid1 * aid2).sum(dim=1)\n    \nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self, name, fmt=':f'):\n        self.name = name\n        self.fmt = fmt\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n    def __str__(self):\n        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n        return fmtstr.format(**self.__dict__)\n\nvalid_ds = Dataset('valid_pairs.parquet')\nvalid_dl_merlin = Loader(valid_ds, 65536, True)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:56:54.278626Z","iopub.execute_input":"2023-03-14T18:56:54.278994Z","iopub.status.idle":"2023-03-14T18:56:54.418614Z","shell.execute_reply.started":"2023-03-14T18:56:54.278964Z","shell.execute_reply":"2023-03-14T18:56:54.417539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs=100\nlr=0.1\n\nmodel = MatrixFactorization(cardinality_aids+1, 32)\noptimizer = SparseAdam(model.parameters(), lr=lr)\ncriterion = nn.BCEWithLogitsLoss()\n\ndevice = torch.device(\"cuda\")\nmodel.cuda()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:31:18.438357Z","iopub.execute_input":"2023-03-14T17:31:18.438726Z","iopub.status.idle":"2023-03-14T17:31:18.596964Z","shell.execute_reply.started":"2023-03-14T17:31:18.438694Z","shell.execute_reply":"2023-03-14T17:31:18.595883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor epoch in range(num_epochs):\n    for batch, _ in train_dl_merlin:\n        model.train()\n        losses = AverageMeter('Loss', ':.4e')\n            \n        aid1, aid2 = batch['aid'], batch['aid_next']\n        output_pos = model(aid1, aid2)\n        output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n        \n        output = torch.cat([output_pos, output_neg])\n        targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])\n        loss = criterion(output, targets)\n        losses.update(loss.item())\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    model.eval()\n    \n    with torch.no_grad():\n        accuracy = AverageMeter('accuracy')\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'], batch['aid_next']\n            output_pos = model(aid1, aid2)\n            output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n            accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()\n            accuracy.update(accuracy_batch, aid1.shape[0])\n    \n    print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:31:20.894171Z","iopub.execute_input":"2023-03-14T17:31:20.894537Z","iopub.status.idle":"2023-03-14T17:31:39.878193Z","shell.execute_reply.started":"2023-03-14T17:31:20.894500Z","shell.execute_reply":"2023-03-14T17:31:39.877132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's grab the embeddings!","metadata":{}},{"cell_type":"code","source":"embeddings = model.cpu().aid_factors.weight.detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:35:07.075104Z","iopub.execute_input":"2023-03-14T17:35:07.075613Z","iopub.status.idle":"2023-03-14T17:35:07.175735Z","shell.execute_reply.started":"2023-03-14T17:35:07.075569Z","shell.execute_reply":"2023-03-14T17:35:07.174559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And create the index for approximate nearest neighbor search.","metadata":{}},{"cell_type":"code","source":"%%time\n\nfrom annoy import AnnoyIndex\n\nindex = AnnoyIndex(32, 'euclidean')\nfor i, v in enumerate(embeddings):\n    index.add_item(i, v)\n    \nindex.build(100)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:35:10.725690Z","iopub.execute_input":"2023-03-14T17:35:10.726065Z","iopub.status.idle":"2023-03-14T17:35:53.339149Z","shell.execute_reply.started":"2023-03-14T17:35:10.726033Z","shell.execute_reply":"2023-03-14T17:35:53.337949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now for any `aid`, we can find its nearest neighbor!","metadata":{}},{"cell_type":"code","source":"index.get_nns_by_item(123, 10)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:36:23.982944Z","iopub.execute_input":"2023-03-14T17:36:23.983298Z","iopub.status.idle":"2023-03-14T17:36:23.990776Z","shell.execute_reply.started":"2023-03-14T17:36:23.983271Z","shell.execute_reply":"2023-03-14T17:36:23.989850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can get the similarity between 2 `aids` as well","metadata":{}},{"cell_type":"code","source":"index.get_distance(123,101041)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:36:27.484310Z","iopub.execute_input":"2023-03-14T17:36:27.485265Z","iopub.status.idle":"2023-03-14T17:36:27.491949Z","shell.execute_reply.started":"2023-03-14T17:36:27.485227Z","shell.execute_reply":"2023-03-14T17:36:27.490867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create MF similarities below:","metadata":{}},{"cell_type":"code","source":"for type_str in tqdm(['clicks','carts_orders','buy2buy']): # types\n    temp_df = pd.read_parquet(f'/kaggle/input/otto-surrogate-feature-generation/covisitation/{RUN_FOR}_top_50_{type_str}.pqt')\n    temp_df['similarity'] = temp_df.apply(lambda x: index.get_distance(int(x['aid_x']),int(x['aid_y'])),axis=1)\n    temp_df.to_parquet(f'/kaggle/working/mf-covisitation/{RUN_FOR}_top_50_{type_str}_mf_similarity.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T17:38:06.549792Z","iopub.execute_input":"2023-03-14T17:38:06.550740Z","iopub.status.idle":"2023-03-14T17:38:19.502812Z","shell.execute_reply.started":"2023-03-14T17:38:06.550705Z","shell.execute_reply":"2023-03-14T17:38:19.501822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n <h1  id=\"2\" style=\"color:white;\n           padding:10px;\n           display:fill;\n           border-radius:10px;\n           background-color:#5642C5;\n           font-size:200%;\n           font-family:Verdana;\n           letter-spacing:0.5px\">  2) Matrix Factorization Features for Test </h1>\n","metadata":{}},{"cell_type":"code","source":"%%time\n\nRUN_FOR = 'kaggle'\ntrain = pl.read_parquet('/kaggle/input/convert-ecommerce-data-to-otto-format/train.parquet') \ntest = pl.read_parquet('/kaggle/input/convert-ecommerce-data-to-otto-format/test.parquet')\n\n\ntrain_pairs = (pl.concat([train, test])\n    .sort(['session','timestamp'])\n    .groupby('session').agg([\n        pl.col('aid'),\n        pl.col('aid').shift(-1).alias('aid_next')\n    ])\n    .explode(['aid', 'aid_next'])\n    .drop_nulls()\n)[['aid', 'aid_next']]\n\ncardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:56:04.360260Z","iopub.execute_input":"2023-03-14T18:56:04.360621Z","iopub.status.idle":"2023-03-14T18:56:06.743972Z","shell.execute_reply.started":"2023-03-14T18:56:04.360591Z","shell.execute_reply":"2023-03-14T18:56:06.742053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_pairs)","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:56:07.581565Z","iopub.execute_input":"2023-03-14T18:56:07.582015Z","iopub.status.idle":"2023-03-14T18:56:07.603595Z","shell.execute_reply.started":"2023-03-14T18:56:07.581976Z","shell.execute_reply":"2023-03-14T18:56:07.602583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs[:-800_000].to_pandas().to_parquet('train_pairs.parquet')\ntrain_pairs[-800_000:].to_pandas().to_parquet('valid_pairs.parquet')\n\ntrain_ds = Dataset('train_pairs.parquet')\ntrain_dl_merlin = Loader(train_ds, 65536, True)\n\nnum_epochs=100\nlr=0.1\n\nmodel = MatrixFactorization(cardinality_aids+1, 32)\noptimizer = SparseAdam(model.parameters(), lr=lr)\ncriterion = nn.BCEWithLogitsLoss()\n\ndevice = torch.device(\"cuda\")\nmodel.cuda()","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:57:08.547288Z","iopub.execute_input":"2023-03-14T18:57:08.547667Z","iopub.status.idle":"2023-03-14T18:57:08.905035Z","shell.execute_reply.started":"2023-03-14T18:57:08.547637Z","shell.execute_reply":"2023-03-14T18:57:08.903688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor epoch in range(num_epochs):\n    for batch, _ in train_dl_merlin:\n        model.train()\n        losses = AverageMeter('Loss', ':.4e')\n            \n        aid1, aid2 = batch['aid'], batch['aid_next']\n        output_pos = model(aid1, aid2)\n        output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n        \n        output = torch.cat([output_pos, output_neg])\n        targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])\n        loss = criterion(output, targets)\n        losses.update(loss.item())\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    model.eval()\n    \n    with torch.no_grad():\n        accuracy = AverageMeter('accuracy')\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'], batch['aid_next']\n            output_pos = model(aid1, aid2)\n            output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n            accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()\n            accuracy.update(accuracy_batch, aid1.shape[0])\n    \n    print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T18:57:11.981064Z","iopub.execute_input":"2023-03-14T18:57:11.981808Z","iopub.status.idle":"2023-03-14T18:57:30.300725Z","shell.execute_reply.started":"2023-03-14T18:57:11.981767Z","shell.execute_reply":"2023-03-14T18:57:30.299551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings = model.cpu().aid_factors.weight.detach().numpy()\nfrom annoy import AnnoyIndex\n\nindex = AnnoyIndex(32, 'euclidean')\nfor i, v in enumerate(embeddings):\n    index.add_item(i, v)\n    \nindex.build(100)\n\nfor type_str in tqdm(['clicks','carts_orders','buy2buy']): # types\n    temp_df = pd.read_parquet(f'/kaggle/input/otto-surrogate-covisitation/covisitation/{RUN_FOR}_top_50_{type_str}.pqt')\n    temp_df['similarity'] = temp_df.apply(lambda x: index.get_distance(int(x['aid_x']),int(x['aid_y'])),axis=1)\n    temp_df.to_parquet(f'/kaggle/working/mf-covisitation/{RUN_FOR}_top_50_{type_str}_mf_similarity.pqt')","metadata":{"execution":{"iopub.status.busy":"2023-03-14T19:24:59.166856Z","iopub.execute_input":"2023-03-14T19:24:59.167428Z","iopub.status.idle":"2023-03-14T19:25:57.032448Z","shell.execute_reply.started":"2023-03-14T19:24:59.167390Z","shell.execute_reply":"2023-03-14T19:25:57.031342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}