{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction","metadata":{}},{"cell_type":"markdown","source":"**Matrix Factorization Model**\n\n1. [Create dataset](https://www.kaggle.com/code/cafelatte1/otto-create-dataset-matrix-factorization)\n2. [Training](https://www.kaggle.com/code/cafelatte1/otto-training-matrix-factorization)\n3. [Inference](https://www.kaggle.com/code/cafelatte1/otto-inference-matrix-factorization)","metadata":{}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"!pip install -q merlin-dataloader==0.0.2","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:49:29.479590Z","iopub.execute_input":"2023-01-16T00:49:29.480080Z","iopub.status.idle":"2023-01-16T00:50:57.571068Z","shell.execute_reply.started":"2023-01-16T00:49:29.479985Z","shell.execute_reply":"2023-01-16T00:50:57.569897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GLOBAL_SEED = 42\n\nimport os\nos.environ['PYTHONHASHSEED'] = str(GLOBAL_SEED)\nimport sys\n\nimport pandas as pd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport random as rnd\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport copy\nimport pickle\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import roc_auc_score\n\nimport cudf\nimport cupy\nfrom cuml.neighbors import NearestNeighbors\n\nfrom merlin.io import Dataset\nfrom merlin.loader.torch import Loader\n\nimport torch\nfrom torch import nn\nfrom torch.optim import SparseAdam\nfrom transformers import get_polynomial_decay_schedule_with_warmup","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:50:57.574730Z","iopub.execute_input":"2023-01-16T00:50:57.575059Z","iopub.status.idle":"2023-01-16T00:51:07.487747Z","shell.execute_reply.started":"2023-01-16T00:50:57.575026Z","shell.execute_reply":"2023-01-16T00:51:07.486733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # RAPIDS random\n    try:\n        cupy.random.seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.manual_seed(seed)\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"w\"):\n    if op==\"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op==\"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n\ndef create_get_ts(ts):\n    return int((ts.replace(tzinfo=CFG.tz) - CFG.ts_zero).total_seconds())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:51:07.492407Z","iopub.execute_input":"2023-01-16T00:51:07.494710Z","iopub.status.idle":"2023-01-16T00:51:07.508798Z","shell.execute_reply.started":"2023-01-16T00:51:07.494672Z","shell.execute_reply":"2023-01-16T00:51:07.507878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    local = False\n    debug = False\n    tz = datetime.timezone.utc\n    ts_zero = datetime.datetime(1970, 1, 1, tzinfo=tz)\n    contentType_mapper = pd.Series([\"click\", \"cart\", \"order\"], index=[0, 1, 2])\n    target_weight = (0.1, 0.3, 0.6)\n    \n    epochs = 1000\n    early_stopping_rounds = epochs // 5\n    eta = 5e-4\n    weight_decay = 1e-4\n\nif CFG.local:\n    CFG.folder_path = \"./dataset/\"\nelse:\n    CFG.folder_path = \"/kaggle/input/\"\n    \nif CFG.debug:\n    CFG.epochs = 5","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:51:07.514109Z","iopub.execute_input":"2023-01-16T00:51:07.516701Z","iopub.status.idle":"2023-01-16T00:51:07.530063Z","shell.execute_reply.started":"2023-01-16T00:51:07.516662Z","shell.execute_reply":"2023-01-16T00:51:07.529051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.train_dates = (create_get_ts(datetime.datetime(2022, 8, 15, 0, 0)), create_get_ts(datetime.datetime(2022, 8, 22, 0, 0)))\nCFG.valid_dates = (create_get_ts(datetime.datetime(2022, 8, 22, 0, 0)), create_get_ts(datetime.datetime(2022, 8, 29, 0, 0)))","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:51:07.534363Z","iopub.execute_input":"2023-01-16T00:51:07.536802Z","iopub.status.idle":"2023-01-16T00:51:07.544469Z","shell.execute_reply.started":"2023-01-16T00:51:07.536768Z","shell.execute_reply":"2023-01-16T00:51:07.543496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"metadata = pickleIO(None, \"/kaggle/input/otto-create-dataset-matrix-factorization/metadata.pkl\", \"r\")\n\ntrain_ds = Dataset(\"/kaggle/input/otto-create-dataset-matrix-factorization/train_pairs.parquet\")\ntrain_dl_merlin = Loader(train_ds, batch_size=int(1024 * 256), shuffle=True)\n\nvalid_ds = Dataset(\"/kaggle/input/otto-create-dataset-matrix-factorization/valid_pairs.parquet\")\nvalid_dl_merlin = Loader(valid_ds, batch_size=int(1024 * 256), shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:51:07.548508Z","iopub.execute_input":"2023-01-16T00:51:07.551144Z","iopub.status.idle":"2023-01-16T00:51:27.809267Z","shell.execute_reply.started":"2023-01-16T00:51:07.551101Z","shell.execute_reply":"2023-01-16T00:51:27.808274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Factorization Model & Define Helper Functions","metadata":{}},{"cell_type":"code","source":"def get_optimizer_params(model, eta, weight_decay):\n    param_optimizer = list(model.named_parameters())\n    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n    optimizer_parameters = [\n        # apply weight decay\n        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n         'lr': eta, 'weight_decay': weight_decay},\n        # don't apply weight decay for LayerNormalization layer\n        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n         'lr': eta, 'weight_decay': 0.0},\n    ]\n    return optimizer_parameters\n\n\ndef get_scheduler(optimizer, num_warmup_steps, num_training_steps):\n    scheduler = get_polynomial_decay_schedule_with_warmup(\n        optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, power=0.9, lr_end=1e-7\n    )\n    return scheduler\n\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self, name, fmt=':f'):\n        self.name = name\n        self.fmt = fmt\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n    def __str__(self):\n        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n        return fmtstr.format(**self.__dict__)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:52:45.705381Z","iopub.execute_input":"2023-01-16T00:52:45.705781Z","iopub.status.idle":"2023-01-16T00:52:45.717223Z","shell.execute_reply.started":"2023-01-16T00:52:45.705749Z","shell.execute_reply":"2023-01-16T00:52:45.716270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MatrixFactorization(nn.Module):\n    def __init__(self, n_aids, embed_size):\n        super().__init__()\n        self.aid_factors = nn.Embedding(n_aids, embed_size, sparse=True)\n        \n    def forward(self, aid1, aid2):\n        aid1 = self.aid_factors(aid1).squeeze(axis=1)\n        aid2 = self.aid_factors(aid2).squeeze(axis=1)\n        return (aid1 * aid2).sum(dim=-1)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:52:47.548832Z","iopub.execute_input":"2023-01-16T00:52:47.549497Z","iopub.status.idle":"2023-01-16T00:52:47.555593Z","shell.execute_reply.started":"2023-01-16T00:52:47.549461Z","shell.execute_reply":"2023-01-16T00:52:47.554687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"model = MatrixFactorization(metadata[\"cardinality_aids\"], 32).to('cuda')\noptimizer_parameters = get_optimizer_params(\n    model,\n    eta=CFG.eta,\n    weight_decay=CFG.weight_decay\n)\noptimizer = SparseAdam(optimizer_parameters, lr=CFG.eta)\nscheduler = get_scheduler(\n    optimizer,\n    num_warmup_steps=(len(train_dl_merlin) * CFG.epochs) // 10,\n    num_training_steps=(len(train_dl_merlin) * CFG.epochs)\n)\ngrad_scaler = torch.cuda.amp.GradScaler()\ncriterion = nn.BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:52:49.148961Z","iopub.execute_input":"2023-01-16T00:52:49.149320Z","iopub.status.idle":"2023-01-16T00:52:49.701838Z","shell.execute_reply.started":"2023-01-16T00:52:49.149291Z","shell.execute_reply":"2023-01-16T00:52:49.700773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fn_training():\n    return_score_dic = []\n    best_score = np.inf\n    \n    seed_everything()\n    for epoch in range(CFG.epochs):\n        train_loss = AverageMeter('Loss', ':.4e')\n        valid_loss = AverageMeter('Loss', ':.4e')\n        train_accuracy = AverageMeter('Accuracy', ':.4e')\n        valid_accuracy = AverageMeter('Accuracy', ':.4e')\n        \n        model.train()\n        for batch, _ in train_dl_merlin:\n            with torch.cuda.amp.autocast():\n                aid1, aid2 = batch['aid'].to('cuda'), batch['aid_next'].to('cuda')\n                output_pos = model(aid1, aid2)\n                output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n                output = torch.cat([output_pos, output_neg])\n                targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_neg)])\n                loss = criterion(output, targets)\n\n            # initialization gradients to zero\n            optimizer.zero_grad()\n            # get scaled gradients by float16 (default)\n            grad_scaler.scale(loss).backward()\n            # apply original gradients (unscaling) to parameters\n            # if these gradients do not contain infs or NaNs, optimizer.step() is then called.\n            # otherwise, optimizer.step() is skipped.\n            grad_scaler.step(optimizer)\n            grad_scaler.update()\n            scheduler.step()\n            \n            train_loss.update(loss.item())    \n            train_accuracy.update(torch.cat([output_pos.flatten().sigmoid() >= 0.5, output_neg.flatten().sigmoid() < 0.5]).float().mean().item())        \n\n        model.eval()\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'].to('cuda'), batch['aid_next'].to('cuda')\n            with torch.no_grad():\n                output_pos = model(aid1, aid2)\n                output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n                output = torch.cat([output_pos, output_neg])\n                targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_neg)])\n                loss = criterion(output, targets)\n\n            valid_loss.update(loss.item())    \n            valid_accuracy.update(torch.cat([output_pos.flatten().sigmoid() >= 0.5, output_neg.flatten().sigmoid() < 0.5]).float().mean().item())        \n\n        score = valid_loss.avg\n        return_score_dic.append({\n            \"epoch\": epoch,\n            \"train_loss\": train_loss.avg,\n            \"valid_loss\": valid_loss.avg,\n            \"train_accuracy\": train_accuracy.avg,\n            \"valid_accuracy\": valid_accuracy.avg,        \n        })\n\n        if score < best_score:\n            best_score = copy.deepcopy(score)\n            model_save_dic = {'model': model.state_dict()}\n            early_stopping_cnt = 0 \n        else:\n            early_stopping_cnt += 1\n\n        if early_stopping_cnt == CFG.early_stopping_rounds:\n            print(\"INFO : Early Stopped ! (Epoch[{0}/{1}])\".format(epoch+1, CFG.epochs))  \n            break\n\n        print(f'[{epoch+1:02d}/{CFG.epochs}]:  * Train Loss {train_loss.avg:.3f} * Train Accuracy {train_accuracy.avg:.3f}  * Valid Loss {valid_loss.avg:.3f} * Valid Accuracy {valid_accuracy.avg:.3f}')\n\n    torch.save(model_save_dic, \"./model_factorization.pth\")\n    return return_score_dic","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:52:58.844603Z","iopub.execute_input":"2023-01-16T00:52:58.845241Z","iopub.status.idle":"2023-01-16T00:52:58.871037Z","shell.execute_reply.started":"2023-01-16T00:52:58.845174Z","shell.execute_reply":"2023-01-16T00:52:58.870147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nscore_dic = fn_training()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:53:00.692269Z","iopub.execute_input":"2023-01-16T00:53:00.692620Z","iopub.status.idle":"2023-01-16T00:54:50.919017Z","shell.execute_reply.started":"2023-01-16T00:53:00.692591Z","shell.execute_reply":"2023-01-16T00:54:50.918168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(score_dic).to_csv(\"./score.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T00:54:50.923282Z","iopub.execute_input":"2023-01-16T00:54:50.925399Z","iopub.status.idle":"2023-01-16T00:54:51.126054Z","shell.execute_reply.started":"2023-01-16T00:54:50.925363Z","shell.execute_reply":"2023-01-16T00:54:51.123848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}