{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is Notebook has been copied from Radek's notebook: [💡Matrix Factorization [PyTorch+Merlin Dataloader]\n](http://www.kaggle.com/code/radek1/matrix-factorization-pytorch-merlin-dataloader)\n\nIn this book I experimented with varying the learning rate using Cosine Annealing with Warm Restarts. I increased the number of Annoy Trees to 200 and increased the epochs to 2. This resulted in a 0.01 increase from Radek's original notebook (0.494 -> 0.504). ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Original Notebook ","metadata":{}},{"cell_type":"code","source":"!pip install pickle5","metadata":{"execution":{"iopub.status.busy":"2023-01-06T17:53:25.454992Z","iopub.execute_input":"2023-01-06T17:53:25.455452Z","iopub.status.idle":"2023-01-06T17:53:36.561060Z","shell.execute_reply.started":"2023-01-06T17:53:25.455398Z","shell.execute_reply":"2023-01-06T17:53:36.559794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom collections import defaultdict\nimport pickle5 as p ","metadata":{"execution":{"iopub.status.busy":"2023-01-06T17:53:36.564064Z","iopub.execute_input":"2023-01-06T17:53:36.565247Z","iopub.status.idle":"2023-01-06T17:53:36.580346Z","shell.execute_reply.started":"2023-01-06T17:53:36.565192Z","shell.execute_reply":"2023-01-06T17:53:36.579180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install polars\n\nimport polars as pl\n\ntrain = pl.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-06T17:53:43.309088Z","iopub.execute_input":"2023-01-06T17:53:43.309902Z","iopub.status.idle":"2023-01-06T17:54:12.250244Z","shell.execute_reply.started":"2023-01-06T17:53:43.309865Z","shell.execute_reply":"2023-01-06T17:54:12.248454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2023-01-06T17:54:56.704519Z","iopub.execute_input":"2023-01-06T17:54:56.704961Z","iopub.status.idle":"2023-01-06T17:54:56.713874Z","shell.execute_reply.started":"2023-01-06T17:54:56.704926Z","shell.execute_reply":"2023-01-06T17:54:56.712476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-06T17:54:12.252232Z","iopub.execute_input":"2023-01-06T17:54:12.252677Z","iopub.status.idle":"2023-01-06T17:54:12.264376Z","shell.execute_reply.started":"2023-01-06T17:54:12.252632Z","shell.execute_reply":"2023-01-06T17:54:12.263109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to create `aid-aid` pairs to train our matrix factorization model.\n\nGrabbing the pairs both from the train and test set.","metadata":{}},{"cell_type":"code","source":"%%time\n\ntrain_pairs = (pl.concat([train, test])\n    .groupby('session').agg([\n        pl.col('aid'),\n        pl.col('aid').shift(-1).alias('aid_next')\n    ])\n    .explode(['aid', 'aid_next'])\n    .drop_nulls()\n)[['aid', 'aid_next']]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:52:54.457005Z","iopub.execute_input":"2023-01-05T19:52:54.457399Z","iopub.status.idle":"2023-01-05T19:53:31.985513Z","shell.execute_reply.started":"2023-01-05T19:52:54.457366Z","shell.execute_reply":"2023-01-05T19:53:31.980463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:54:32.521861Z","iopub.execute_input":"2023-01-05T19:54:32.522450Z","iopub.status.idle":"2023-01-05T19:54:32.532657Z","shell.execute_reply.started":"2023-01-05T19:54:32.522406Z","shell.execute_reply":"2023-01-05T19:54:32.531541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:54:34.724947Z","iopub.execute_input":"2023-01-05T19:54:34.725375Z","iopub.status.idle":"2023-01-05T19:54:34.732385Z","shell.execute_reply.started":"2023-01-05T19:54:34.725338Z","shell.execute_reply":"2023-01-05T19:54:34.731554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Retrieving the cardinality of our aids -- we will need this to create the embedding layer.","metadata":{}},{"cell_type":"code","source":"cardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())\ncardinality_aids","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:54:37.448580Z","iopub.execute_input":"2023-01-05T19:54:37.449621Z","iopub.status.idle":"2023-01-05T19:54:38.343270Z","shell.execute_reply.started":"2023-01-05T19:54:37.449582Z","shell.execute_reply":"2023-01-05T19:54:38.342020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using Merlin Dataloader to load the data","metadata":{}},{"cell_type":"code","source":"!pip install merlin-dataloader==0.0.2","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-05T19:54:40.617828Z","iopub.execute_input":"2023-01-05T19:54:40.618384Z","iopub.status.idle":"2023-01-05T19:56:28.098058Z","shell.execute_reply.started":"2023-01-05T19:54:40.618334Z","shell.execute_reply":"2023-01-05T19:56:28.096432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pairs[:-10_000_000].to_pandas().to_parquet('train_pairs.parquet')\ntrain_pairs[-10_000_000:].to_pandas().to_parquet('valid_pairs.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:56:28.100496Z","iopub.execute_input":"2023-01-05T19:56:28.100913Z","iopub.status.idle":"2023-01-05T19:56:40.872911Z","shell.execute_reply.started":"2023-01-05T19:56:28.100868Z","shell.execute_reply":"2023-01-05T19:56:40.871729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from merlin.loader.torch import Loader \nfrom merlin.io import Dataset\n\ntrain_ds = Dataset('train_pairs.parquet')\ntrain_dl_merlin = Loader(train_ds, 65536, True)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:56:40.874313Z","iopub.execute_input":"2023-01-05T19:56:40.874692Z","iopub.status.idle":"2023-01-05T19:56:44.859542Z","shell.execute_reply.started":"2023-01-05T19:56:40.874657Z","shell.execute_reply":"2023-01-05T19:56:44.858394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Testing how long it takes to iterate over the dataset using dataloader:","metadata":{}},{"cell_type":"code","source":"%%time\n\nfor batch, _ in train_dl_merlin:\n    aid1, aid2 = batch['aid'], batch['aid_next']","metadata":{"execution":{"iopub.status.busy":"2023-01-05T17:45:50.017806Z","iopub.execute_input":"2023-01-05T17:45:50.018348Z","iopub.status.idle":"2023-01-05T17:46:30.539255Z","shell.execute_reply.started":"2023-01-05T17:45:50.018309Z","shell.execute_reply":"2023-01-05T17:46:30.538211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Original MF model without modifications","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\n\nclass MatrixFactorization(nn.Module):\n    def __init__(self, n_aids, n_factors):\n        super().__init__()\n        self.aid_factors = nn.Embedding(n_aids, n_factors, sparse=True)\n        \n    def forward(self, aid1, aid2):\n        aid1 = self.aid_factors(aid1)\n        aid2 = self.aid_factors(aid2)\n        \n        return (aid1 * aid2).sum(dim=1)\n    \nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self, name, fmt=':f'):\n        self.name = name\n        self.fmt = fmt\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n    def __str__(self):\n        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n        return fmtstr.format(**self.__dict__)\n\nvalid_ds = Dataset('valid_pairs.parquet')\nvalid_dl_merlin = Loader(valid_ds, 65536, True)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T19:57:11.961276Z","iopub.execute_input":"2023-01-05T19:57:11.961716Z","iopub.status.idle":"2023-01-05T19:57:12.228075Z","shell.execute_reply.started":"2023-01-05T19:57:11.961675Z","shell.execute_reply":"2023-01-05T19:57:12.226918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim import SparseAdam\n\nnum_epochs=1\nlr=0.1\n\nmodel = MatrixFactorization(cardinality_aids+1, 32)\noptimizer = SparseAdam(model.parameters(), lr=lr)\ncriterion = nn.BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:26:26.484179Z","iopub.execute_input":"2023-01-04T19:26:26.484601Z","iopub.status.idle":"2023-01-04T19:26:27.030000Z","shell.execute_reply.started":"2023-01-04T19:26:26.484572Z","shell.execute_reply":"2023-01-04T19:26:27.029033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor epoch in range(num_epochs):\n    for batch, _ in train_dl_merlin:\n        model.train()\n        losses = AverageMeter('Loss', ':.4e')\n            \n        aid1, aid2 = batch['aid'], batch['aid_next']\n        output_pos = model(aid1, aid2)\n        output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n        \n        output = torch.cat([output_pos, output_neg])\n        targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])\n        loss = criterion(output, targets)\n        losses.update(loss.item())\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    model.eval()\n    \n    with torch.no_grad():\n        accuracy = AverageMeter('accuracy')\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'], batch['aid_next']\n            output_pos = model(aid1, aid2)\n            output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n            accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()\n            accuracy.update(accuracy_batch, aid1.shape[0])\n            \n    print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:28:05.946335Z","iopub.execute_input":"2023-01-04T19:28:05.946698Z","iopub.status.idle":"2023-01-04T19:49:49.938991Z","shell.execute_reply.started":"2023-01-04T19:28:05.946672Z","shell.execute_reply":"2023-01-04T19:49:49.938072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's grab the embeddings!","metadata":{}},{"cell_type":"code","source":"embeddings = model.aid_factors.weight.detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:50:52.029050Z","iopub.execute_input":"2023-01-04T19:50:52.029399Z","iopub.status.idle":"2023-01-04T19:50:52.034583Z","shell.execute_reply.started":"2023-01-04T19:50:52.029374Z","shell.execute_reply":"2023-01-04T19:50:52.033524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And create the index for approximate nearest neighbor search.","metadata":{}},{"cell_type":"code","source":"%%time\n\nfrom annoy import AnnoyIndex\n\nindex = AnnoyIndex(32, 'euclidean')\nfor i, v in enumerate(embeddings):\n    index.add_item(i, v)\n    \nindex.build(100)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:51:06.499573Z","iopub.execute_input":"2023-01-04T19:51:06.499926Z","iopub.status.idle":"2023-01-04T19:53:01.940942Z","shell.execute_reply.started":"2023-01-04T19:51:06.499900Z","shell.execute_reply":"2023-01-04T19:53:01.940319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now for any `aid`, we can find its nearest neighbor!","metadata":{}},{"cell_type":"code","source":"index.get_nns_by_item(123, 10)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:53:46.040398Z","iopub.execute_input":"2023-01-04T19:53:46.041509Z","iopub.status.idle":"2023-01-04T19:53:46.047863Z","shell.execute_reply.started":"2023-01-04T19:53:46.041477Z","shell.execute_reply":"2023-01-04T19:53:46.046854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a submission below:","metadata":{}},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/otto-recommender-system//sample_submission.csv')\n\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = test.to_pandas().reset_index(drop=True).groupby('session')['aid'].apply(list)\ntest_session_types = test.to_pandas().reset_index(drop=True).groupby('session')['type'].apply(list)\n\ntest_session_AIDs","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:53:55.420284Z","iopub.execute_input":"2023-01-04T19:53:55.420634Z","iopub.status.idle":"2023-01-04T19:54:55.077576Z","shell.execute_reply.started":"2023-01-04T19:53:55.420608Z","shell.execute_reply":"2023-01-04T19:54:55.076496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_session_types.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:55:13.338892Z","iopub.execute_input":"2023-01-04T19:55:13.339295Z","iopub.status.idle":"2023-01-04T19:55:13.348003Z","shell.execute_reply.started":"2023-01-04T19:55:13.339266Z","shell.execute_reply":"2023-01-04T19:55:13.347037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type2id_path = '../input/otto-full-optimized-memory-footprint/type2id.pkl'\n\n\nwith open(type2id_path, \"rb\") as fh:\n    data = p.load(fh)\n\nprint('Showing the pickled data:')\n\ncnt = 0\nfor item in data:\n    print('The data ', cnt, ' is : ', item)\n    cnt += 1","metadata":{"execution":{"iopub.status.busy":"2023-01-04T20:03:36.313140Z","iopub.execute_input":"2023-01-04T20:03:36.313479Z","iopub.status.idle":"2023-01-04T20:03:36.320026Z","shell.execute_reply.started":"2023-01-04T20:03:36.313453Z","shell.execute_reply":"2023-01-04T20:03:36.319035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\n\ntype_weight_multipliers = {0: 1, 1: 6, 2: 3}\nfor AIDs, types in zip(test_session_AIDs, test_session_types):\n    if len(AIDs) >= 20:\n        '''If we have enough test aids (>20) we don't need to look for candidates. \n        First we initialize weights so that more recent AIDs have a higher weight. The weights diminish \n        based on numbers evenly spaced on a log scale. Then the weights are multiplied by type_weight_multipliers\n        if they are carts or orders.''' \n        weights=np.logspace(0.1,1,len(AIDs),base=2, endpoint=True)-1\n        aids_temp=defaultdict(lambda: 0)\n        for aid,w,t in zip(AIDs,weights,types): \n            aids_temp[aid]+= w * type_weight_multipliers[t]\n            \n        sorted_aids=[k for k, v in sorted(aids_temp.items(), key=lambda item: -item[1])]\n        labels.append(sorted_aids[:20])\n    else:\n        #When we don't have 20 AIDs in the test data, we use nearest neighbour to search for candidates\n        #We create a dictionary from test data to get unique AIDs\n        AIDs = list(dict.fromkeys(AIDs[::-1]))\n        \n        # let's grab the most recent aid\n        most_recent_aid = AIDs[0]\n        \n        # and look for some neighbors based on our embeddings \n        nns = index.get_nns_by_item(most_recent_aid, 21)[1:]\n                        \n        labels.append((AIDs+nns)[:20])","metadata":{"execution":{"iopub.status.busy":"2023-01-04T20:47:52.474706Z","iopub.execute_input":"2023-01-04T20:47:52.475992Z","iopub.status.idle":"2023-01-04T21:01:39.980975Z","shell.execute_reply.started":"2023-01-04T20:47:52.475942Z","shell.execute_reply":"2023-01-04T21:01:39.979976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:07:26.309192Z","iopub.execute_input":"2023-01-04T21:07:26.310489Z","iopub.status.idle":"2023-01-04T21:07:26.320571Z","shell.execute_reply.started":"2023-01-04T21:07:26.310448Z","shell.execute_reply":"2023-01-04T21:07:26.319517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:07:26.373147Z","iopub.execute_input":"2023-01-04T21:07:26.375126Z","iopub.status.idle":"2023-01-04T21:07:26.383205Z","shell.execute_reply.started":"2023-01-04T21:07:26.375096Z","shell.execute_reply":"2023-01-04T21:07:26.382125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_as_strings = [' '.join([str(l) for l in lls]) for lls in labels]\n\npredictions = pd.DataFrame(data={'session_type': test_session_AIDs.index, 'labels': labels_as_strings})","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:02:07.836451Z","iopub.execute_input":"2023-01-04T21:02:07.836793Z","iopub.status.idle":"2023-01-04T21:02:15.823456Z","shell.execute_reply.started":"2023-01-04T21:02:07.836766Z","shell.execute_reply":"2023-01-04T21:02:15.821999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:03:34.354650Z","iopub.execute_input":"2023-01-04T21:03:34.354984Z","iopub.status.idle":"2023-01-04T21:03:34.402737Z","shell.execute_reply.started":"2023-01-04T21:03:34.354958Z","shell.execute_reply":"2023-01-04T21:03:34.401134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_dfs = []\n\nfor st in session_types:\n    modified_predictions = predictions.copy()\n    modified_predictions.session_type = modified_predictions.session_type.astype('str') + f'_{st}'\n    prediction_dfs.append(modified_predictions)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:09:41.794481Z","iopub.execute_input":"2023-01-04T21:09:41.794943Z","iopub.status.idle":"2023-01-04T21:09:44.928989Z","shell.execute_reply.started":"2023-01-04T21:09:41.794915Z","shell.execute_reply":"2023-01-04T21:09:44.928006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat(prediction_dfs).reset_index(drop=True)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:11:01.234458Z","iopub.execute_input":"2023-01-04T21:11:01.235624Z","iopub.status.idle":"2023-01-04T21:11:16.256048Z","shell.execute_reply.started":"2023-01-04T21:11:01.235593Z","shell.execute_reply":"2023-01-04T21:11:16.255066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T21:11:52.447020Z","iopub.execute_input":"2023-01-04T21:11:52.447435Z","iopub.status.idle":"2023-01-04T21:11:52.455302Z","shell.execute_reply.started":"2023-01-04T21:11:52.447411Z","shell.execute_reply":"2023-01-04T21:11:52.454468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Varying the Learning Rate using Cosine Annealing","metadata":{}},{"cell_type":"markdown","source":"I want to try a method I saw in a [blogpost](https://eugeneyan.com/writing/recommender-systems-baseline-pytorch/) on varying the learning rate using cosine annealing. Demonstrating how it would work below:","metadata":{}},{"cell_type":"code","source":"from torch import optim\nfrom torch.optim import SparseAdam\n\nnum_epochs=5\ninitial_lr=0.1\n\nmodel = MatrixFactorization(cardinality_aids+1, 32)\noptimizer = SparseAdam(model.parameters(), lr=initial_lr)\nscheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, len(train_dl_merlin), eta_min = 0.001)\ncriterion = nn.BCEWithLogitsLoss()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T20:19:54.642986Z","iopub.execute_input":"2023-01-05T20:19:54.643552Z","iopub.status.idle":"2023-01-05T20:19:55.277301Z","shell.execute_reply.started":"2023-01-05T20:19:54.643498Z","shell.execute_reply":"2023-01-05T20:19:55.276405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lrs = []\nfor i in range(num_epochs):\n    for batch, _ in train_dl_merlin:\n        lrs.append(optimizer.state_dict()[\"param_groups\"][0][\"lr\"])\n        optimizer.step()\n        scheduler.step()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T20:19:59.801129Z","iopub.execute_input":"2023-01-05T20:19:59.801599Z","iopub.status.idle":"2023-01-05T20:23:05.903463Z","shell.execute_reply.started":"2023-01-05T20:19:59.801559Z","shell.execute_reply":"2023-01-05T20:23:05.902048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nplt.scatter(range(num_epochs*len(train_dl_merlin)), lrs)\nplt.title(\"Cosine Annealing LR Scheduler\")\nplt.xlabel(\"Batch\")\nplt.ylabel(\"Learning Rate\");","metadata":{"execution":{"iopub.status.busy":"2023-01-05T20:28:04.247775Z","iopub.execute_input":"2023-01-05T20:28:04.248278Z","iopub.status.idle":"2023-01-05T20:28:04.597085Z","shell.execute_reply.started":"2023-01-05T20:28:04.248236Z","shell.execute_reply":"2023-01-05T20:28:04.595767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nnum_epochs=2\ninitial_lr=0.1\nfor epoch in range(num_epochs):\n    \n    for batch, i in train_dl_merlin:\n        \n        model.train()\n        losses = AverageMeter('Loss', ':.4e')\n            \n        aid1, aid2 = batch['aid'], batch['aid_next']\n        output_pos = model(aid1, aid2)\n        output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n        \n        output = torch.cat([output_pos, output_neg])\n        targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])\n        loss = criterion(output, targets)\n        losses.update(loss.item())\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        \n    model.eval()\n    \n    with torch.no_grad():\n        accuracy = AverageMeter('accuracy')\n        for batch, _ in valid_dl_merlin:\n            aid1, aid2 = batch['aid'], batch['aid_next']\n            output_pos = model(aid1, aid2)\n            output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])\n            accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()\n            accuracy.update(accuracy_batch, aid1.shape[0])\n            \n    print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')","metadata":{"execution":{"iopub.status.busy":"2023-01-05T21:06:17.608109Z","iopub.execute_input":"2023-01-05T21:06:17.608903Z","iopub.status.idle":"2023-01-05T22:05:34.009010Z","shell.execute_reply.started":"2023-01-05T21:06:17.608862Z","shell.execute_reply":"2023-01-05T22:05:34.007649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make Predictions for Submission","metadata":{}},{"cell_type":"code","source":"embeddings = model.aid_factors.weight.detach().numpy()\n\nfrom annoy import AnnoyIndex\n\nindex = AnnoyIndex(32, 'euclidean')\nfor i, v in enumerate(embeddings):\n    index.add_item(i, v)\n    \nindex.build(200)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:05:51.602639Z","iopub.execute_input":"2023-01-05T22:05:51.603519Z","iopub.status.idle":"2023-01-05T22:11:01.160405Z","shell.execute_reply.started":"2023-01-05T22:05:51.603441Z","shell.execute_reply":"2023-01-05T22:11:01.159382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index.get_nns_by_item(123, 10)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:44:01.607704Z","iopub.execute_input":"2023-01-05T22:44:01.608177Z","iopub.status.idle":"2023-01-05T22:44:01.617676Z","shell.execute_reply.started":"2023-01-05T22:44:01.608135Z","shell.execute_reply":"2023-01-05T22:44:01.616079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_session_AIDs)/10000","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:45:19.375315Z","iopub.execute_input":"2023-01-05T22:45:19.375750Z","iopub.status.idle":"2023-01-05T22:45:19.383152Z","shell.execute_reply.started":"2023-01-05T22:45:19.375715Z","shell.execute_reply":"2023-01-05T22:45:19.381947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/otto-recommender-system//sample_submission.csv')\n\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = test.to_pandas().reset_index(drop=True).groupby('session')['aid'].apply(list)\ntest_session_types = test.to_pandas().reset_index(drop=True).groupby('session')['type'].apply(list)\n\nlabels = []\n\ntype_weight_multipliers = {0: 1, 1: 6, 2: 3}\ni = 0 \nfor AIDs, types in zip(test_session_AIDs, test_session_types):\n    if len(AIDs) >= 20:\n        '''If we have enough test aids (>20) we don't need to look for candidates. \n        First we initialize weights so that more recent AIDs have a higher weight. The weights diminish \n        based on numbers evenly spaced on a log scale. Then the weights are multiplied by type_weight_multipliers\n        if they are carts or orders.''' \n        weights=np.logspace(0.1,1,len(AIDs),base=2, endpoint=True)-1\n        aids_temp=defaultdict(lambda: 0)\n        for aid,w,t in zip(AIDs,weights,types): \n            aids_temp[aid]+= w * type_weight_multipliers[t]\n            \n        sorted_aids=[k for k, v in sorted(aids_temp.items(), key=lambda item: -item[1])]\n        labels.append(sorted_aids[:20])\n    else:\n        #When we don't have 20 AIDs in the test data, we use nearest neighbour to search for candidates\n        #We create a dictionary from test data to get unique AIDs\n        AIDs = list(dict.fromkeys(AIDs[::-1]))\n        \n        # let's grab the most recent aid\n        most_recent_aid = AIDs[0]\n        \n        # and look for some neighbors based on our embeddings \n        nns = index.get_nns_by_item(most_recent_aid, 21)[1:]\n                        \n        labels.append((AIDs+nns)[:20])\n    i += 1 \n    if i%10000 == 0:\n        print (i)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T22:45:26.993688Z","iopub.execute_input":"2023-01-05T22:45:26.994147Z","iopub.status.idle":"2023-01-05T23:22:23.745392Z","shell.execute_reply.started":"2023-01-05T22:45:26.994108Z","shell.execute_reply":"2023-01-05T23:22:23.743778Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (len(labels))","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:22:28.531658Z","iopub.execute_input":"2023-01-05T23:22:28.532517Z","iopub.status.idle":"2023-01-05T23:22:28.539305Z","shell.execute_reply.started":"2023-01-05T23:22:28.532456Z","shell.execute_reply":"2023-01-05T23:22:28.538196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_as_strings = [' '.join([str(l) for l in lls]) for lls in labels]\n\npredictions = pd.DataFrame(data={'session_type': test_session_AIDs.index, 'labels': labels_as_strings})\n\nprediction_dfs = []\n\nfor st in session_types:\n    modified_predictions = predictions.copy()\n    modified_predictions.session_type = modified_predictions.session_type.astype('str') + f'_{st}'\n    prediction_dfs.append(modified_predictions)\n    \nsubmission = pd.concat(prediction_dfs).reset_index(drop=True)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T23:22:39.654360Z","iopub.execute_input":"2023-01-05T23:22:39.654808Z","iopub.status.idle":"2023-01-05T23:23:15.694353Z","shell.execute_reply.started":"2023-01-05T23:22:39.654772Z","shell.execute_reply":"2023-01-05T23:23:15.693043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adding Bias and L2 Regularization in the model","metadata":{}},{"cell_type":"code","source":"def regularize_l2(array):\n    loss = torch.sum(array ** 2.0)\n    return loss","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}