{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"参考ノート\nhttps://www.kaggle.com/code/radek1/matrix-factorization-pytorch-merlin-dataloader","metadata":{}},{"cell_type":"code","source":"import cudf\n\ntrain = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\n\ntrain_pairs = cudf.concat([train, test])[['session', 'aid']]\ndel train, test\n\n# 次のaidを並べてペアを作成する。\n# 並び順から埋め込みベクトルを作成できる。\n# 自然言語処理で2グラムの並びをモデルに与えると改善することが多い\ntrain_pairs['aid_next'] = train_pairs.groupby('session').aid.shift(-1)\ntrain_pairs = train_pairs[['aid', 'aid_next']].dropna().reset_index(drop=True)\n\n# aidの最大値を出してaidの数を出す\ncardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())\nprint('Cardinality of items is',cardinality_aids)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-03-29T09:08:39.936726Z","iopub.execute_input":"2023-03-29T09:08:39.937319Z","iopub.status.idle":"2023-03-29T09:09:17.278185Z","shell.execute_reply.started":"2023-03-29T09:08:39.937231Z","shell.execute_reply":"2023-03-29T09:09:17.277118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs","metadata":{"execution":{"iopub.status.busy":"2023-03-29T09:10:17.666843Z","iopub.execute_input":"2023-03-29T09:10:17.667280Z","iopub.status.idle":"2023-03-29T09:10:17.775897Z","shell.execute_reply.started":"2023-03-29T09:10:17.667243Z","shell.execute_reply":"2023-03-29T09:10:17.774956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install merlin-dataloader==0.0.2\nfrom merlin.loader.torch import Loader \n\ntrain_pairs.to_pandas().to_parquet('train_pairs.parquet')\n# 1000万行を評価表のaidpairに残す\ntrain_pairs[-10_000_000:].to_pandas().to_parquet('valid_pairs.parquet')\n\nfrom merlin.loader.torch import Loader \nfrom merlin.io import Dataset\n\n# 辞書型で65536ずつのaidを入れる\n# aidの辞書とaid_nextの辞書\ntrain_ds = Dataset('train_pairs.parquet')\ntrain_dl_merlin = Loader(train_ds, 65536, True)","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-03-29T09:12:52.767357Z","iopub.execute_input":"2023-03-29T09:12:52.767832Z","iopub.status.idle":"2023-03-29T09:14:38.499728Z","shell.execute_reply.started":"2023-03-29T09:12:52.767791Z","shell.execute_reply":"2023-03-29T09:14:38.498416Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dl_merlin","metadata":{"execution":{"iopub.status.busy":"2023-03-29T09:17:01.762524Z","iopub.execute_input":"2023-03-29T09:17:01.763030Z","iopub.status.idle":"2023-03-29T09:17:01.770632Z","shell.execute_reply.started":"2023-03-29T09:17:01.762984Z","shell.execute_reply":"2023-03-29T09:17:01.769665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\n\n# nn.ModuleはPyTorchの関数\n# 全てのニューラルネットワークの基礎クラスなので、Pytorch利用の際に必ず入れるルールのようなもの\nclass MatrixFactorization(nn.Module):\n    def __init__(self, n_aids, n_factors):\n        super().__init__()\n        # nn.Embedding(読み込むdfのサイズ、次元サイズ\n        # 整数のインデックスを与えるだけで埋め込みベクトルを返す\n        self.aid_factors = nn.Embedding(n_aids, n_factors, sparse=True)\n        \n    def forward(self, aid1, aid2):\n        aid1 = self.aid_factors(aid1)\n        aid2 = self.aid_factors(aid2)\n        \n        return (aid1 * aid2).sum(dim=1)\n    \nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self, name, fmt=':f'):\n        self.name = name\n        self.fmt = fmt\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n    def __str__(self):\n        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n        return fmtstr.format(**self.__dict__)\n\nvalid_ds = Dataset('valid_pairs.parquet')\nvalid_dl_merlin = Loader(valid_ds, 65536, True)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-03-29T09:27:10.852009Z","iopub.execute_input":"2023-03-29T09:27:10.852377Z","iopub.status.idle":"2023-03-29T09:27:11.000032Z","shell.execute_reply.started":"2023-03-29T09:27:10.852345Z","shell.execute_reply":"2023-03-29T09:27:10.999093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dl_merlin","metadata":{"execution":{"iopub.status.busy":"2023-03-29T09:27:27.164381Z","iopub.execute_input":"2023-03-29T09:27:27.164967Z","iopub.status.idle":"2023-03-29T09:27:27.177753Z","shell.execute_reply.started":"2023-03-29T09:27:27.164922Z","shell.execute_reply":"2023-03-29T09:27:27.176708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim import SparseAdam\n\nnum_epochs = 10\nlr=0.1\n\n# 32次元で行列分解を行う\nmodel = MatrixFactorization(cardinality_aids+1, 32)\noptimizer = SparseAdam(model.parameters(), lr=lr)\ncriterion = nn.BCEWithLogitsLoss()\n\nmodel.to('cuda')\nfor epoch in range(num_epochs):\n    for batch, _ in train_dl_merlin:\n        model.train()\n        losses = AverageMeter('Loss', ':.4e')\n            \n        aid1, aid2 = batch['aid'], batch['aid_next']\n        aid1 = aid1.to('cuda')\n        aid2 = aid2.to('cuda')\n        output_pos = model(aid1, aid2)\n        output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n        \n        output = torch.cat([output_pos, output_neg])\n        targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])\n        loss = criterion(output, targets)\n        losses.update(loss.item())\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    model.eval()\n    \n    with torch.no_grad():\n        accuracy = AverageMeter('accuracy')\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'], batch['aid_next']\n            output_pos = model(aid1, aid2)\n            output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n            accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()\n            accuracy.update(accuracy_batch, aid1.shape[0])\n            \n    print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-01-23T05:47:14.120245Z","iopub.execute_input":"2023-01-23T05:47:14.120627Z","iopub.status.idle":"2023-01-23T05:51:51.266729Z","shell.execute_reply.started":"2023-01-23T05:47:14.120593Z","shell.execute_reply":"2023-01-23T05:51:51.265654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aid数185万*32次元の特徴ベクトル\nembeddings = model.aid_factors.weight.detach().cpu().numpy()\nprint('Item Matrix Factorization embeddings have shape',embeddings.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:51:51.269327Z","iopub.execute_input":"2023-01-23T05:51:51.269632Z","iopub.status.idle":"2023-01-23T05:51:51.463660Z","shell.execute_reply.started":"2023-01-23T05:51:51.269606Z","shell.execute_reply":"2023-01-23T05:51:51.462654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMPORT RAPIDS TSNE\nfrom cuml import UMAP, TSNE, PCA\nimport matplotlib.pyplot as plt, numpy as np\nimport matplotlib.patches as mpatches, cuml\nprint('RAPIDS cuML version',cuml.__version__)\n\n# TSNEで高次元のデータを2次元に削減\n# 特徴ベクトルを2次元に削減\nem_2d = TSNE(n_components=2).fit_transform(embeddings)\nprint('TSNE embeddings have shape',em_2d.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:51:51.465043Z","iopub.execute_input":"2023-01-23T05:51:51.465660Z","iopub.status.idle":"2023-01-23T05:54:26.894561Z","shell.execute_reply.started":"2023-01-23T05:51:51.465623Z","shell.execute_reply":"2023-01-23T05:54:26.893490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nembe_32=pd.DataFrame(embeddings)\nembe_32=embe_32.add_prefix(\"vector_\")\nembe_32=embe_32.reset_index()\nembe_32=embe_32.rename(columns={\"index\":\"aid\"})\nembe_32","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:48.947678Z","iopub.execute_input":"2023-01-23T05:59:48.948682Z","iopub.status.idle":"2023-01-23T05:59:50.426982Z","shell.execute_reply.started":"2023-01-23T05:59:48.948634Z","shell.execute_reply":"2023-01-23T05:59:50.425897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embe_32.to_parquet(\"embe_32.pqt\")","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:53.202236Z","iopub.execute_input":"2023-01-23T05:59:53.202641Z","iopub.status.idle":"2023-01-23T05:59:54.805151Z","shell.execute_reply.started":"2023-01-23T05:59:53.202609Z","shell.execute_reply":"2023-01-23T05:59:54.803734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embe_2=pd.DataFrame(em_2d)\nembe_2=embe_2.add_prefix(\"vector_\")\nembe_2=embe_2.reset_index()\nembe_2=embe_2.rename(columns={\"index\":\"aid\"})\nembe_2","metadata":{"execution":{"iopub.status.busy":"2023-01-23T05:59:55.840054Z","iopub.execute_input":"2023-01-23T05:59:55.840422Z","iopub.status.idle":"2023-01-23T05:59:55.875085Z","shell.execute_reply.started":"2023-01-23T05:59:55.840392Z","shell.execute_reply":"2023-01-23T05:59:55.874046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embe_2.to_parquet(\"embe_2.pqt\")","metadata":{"execution":{"iopub.status.busy":"2023-01-23T06:00:00.125343Z","iopub.execute_input":"2023-01-23T06:00:00.125698Z","iopub.status.idle":"2023-01-23T06:00:00.291970Z","shell.execute_reply.started":"2023-01-23T06:00:00.125668Z","shell.execute_reply":"2023-01-23T06:00:00.290967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"em_2d","metadata":{"execution":{"iopub.status.busy":"2023-01-23T03:03:01.990848Z","iopub.execute_input":"2023-01-23T03:03:01.991489Z","iopub.status.idle":"2023-01-23T03:03:02.002554Z","shell.execute_reply.started":"2023-01-23T03:03:01.991448Z","shell.execute_reply":"2023-01-23T03:03:02.001481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}