{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transformers4rec[pytorch,nvtabular]\n!pip install -U nvtabular==1.3.3\n!pip install -U pytorch_lightning==1.8.0.post1\n!pip install torchmetrics==0.10.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\nimport pickle\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.notebook import tqdm\n\nwarnings.simplefilter(action=\"ignore\", category=FutureWarning)\nwarnings.simplefilter(action=\"ignore\", category=UserWarning)\n\nfrom merlin_standard_lib import Schema, Tag\nfrom transformers4rec import torch as tr\nfrom transformers4rec.torch import Trainer\nfrom transformers4rec.torch.ranking_metric import RecallAt\nfrom transformers4rec.config.trainer import T4RecTrainingArguments\nfrom transformers4rec.torch.utils.data_utils import T4RecDataLoader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 500\nD_MODEL = 64\n# D_MODEL = 320\n\nTRAIN_BATCH_SIZE = 128\nEVAL_BATCH_SIZE = 128\n\n\nif DEBUG:\n    MAX_STEPS = 2000\n    EVAL_BATCH_SIZE = 4\nelse:\n    MAX_STEPS = -1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    train_path = [\n        \"../input/otto-etl-nvtabular-cpu/train/part_0.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_1.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_2.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_3.parquet\",\n    ]\n    valid_path = \"../input/otto-etl-nvtabular-cpu/valid/part_0.parquet\"\n    test_path = \"../input/otto-etl-nvtabular-cpu/test/part_0.parquet\"\n    \nelse:\n    train_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/train/part_*.parquet\"))\n    valid_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/valid/part_*.parquet\"))\n    test_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/test/part_*.parquet\"))\n    \noutput_path = \"checkpoint/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create schema proto manually\n\n(on Oct 6, 2021)\n> The latest version of the NVTabular library (used for the ETL pipeline) automatically outputs a schema file in the protobuf text format (schema.pbtxt) together with the parquet files of processed data. This is still a work in progress and we'll include the necessary support in the future release of Transformers4Rec.\n\nhttps://github.com/NVIDIA-Merlin/Transformers4Rec/issues/271#issuecomment-934850026","metadata":{}},{"cell_type":"code","source":"%%writefile schema.pb\n\nfeature {\n  name: \"session\"\n  type: INT\n  int_domain {\n    name: \"session\"\n    min: 0\n    max: 12899778\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"categorical\"\n  }\n}\n\n\nfeature {\n  name: \"event_type\"\n  type: INT\n  int_domain {\n    name: \"event_type\"\n    min: 1\n    max: 3\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"top_20_clicks-list\"\n  value_count {\n    min: 1\n    max: 20\n  }\n  type: INT\n  int_domain {\n    name: \"top_20_clicks-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"top_15_buy2buy-list\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_buy2buy-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"top_15_carts_orders-list\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_carts_orders-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"aid-list\"\n  value_count {\n    min: 1\n    max: 500\n  }\n  type: INT\n  int_domain {\n    name: \"aid-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"item\"\n    tag: \"categorical\"\n    tag: \"list\"\n    tag: \"item_id\"\n  }\n}\n\nfeature {\n  name: \"top_20_clicks-list_raw\"\n  value_count {\n    min: 1\n    max: 20\n  }\n  type: INT\n  int_domain {\n    name: \"top_20_clicks-list_raw\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"top_15_buy2buy-list_raw\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_buy2buy-list_raw\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"top_15_carts_orders-list_raw\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_carts_orders-list_raw\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"labels_clicks\"\n  type: INT\n  int_domain {\n    name: \"labels_clicks\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"labels_carts\"\n  value_count {\n    min: 1\n    max: 150\n  }\n  type: INT\n  int_domain {\n    name: \"labels_carts\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}\n\nfeature {\n  name: \"labels_orders\"\n  value_count {\n    min: 1\n    max: 100\n  }\n  type: INT\n  int_domain {\n    name: \"labels_orders\"\n    min: 1\n    max: 1855603\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n  }\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"schema_all = Schema().from_proto_text(\"schema.pb\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"schema_train = schema_all.select_by_name(\n    [\n        \"event_type\",\n        \"top_20_clicks-list\",\n        \"top_15_buy2buy-list\",\n        \"top_15_carts_orders-list\",\n        \"aid-list\",\n    ]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"schema_valid = schema_all.select_by_name(\n    [\n        \"session\",\n        \"event_type\",\n        \"top_20_clicks-list\",\n        \"top_15_buy2buy-list\",\n        \"top_15_carts_orders-list\",\n        \"aid-list\",\n        \"labels_clicks\",\n        \"labels_carts\",\n        \"labels_orders\",\n    ]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Model and Trainer","metadata":{}},{"cell_type":"code","source":"inputs = tr.TabularSequenceFeatures.from_schema(\n    schema_train,\n    max_sequence_length=MAX_SEQUENCE_LENGTH,\n#     aggregation=\"concat\",\n    d_output=D_MODEL,\n#     masking=\"mlm\",\n    masking=\"clm\",\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define XLNetConfig class and set default parameters for HF XLNet config  \ntransformer_config = tr.XLNetConfig.build(\n    d_model=D_MODEL, n_head=4, n_layer=2, total_seq_length=MAX_SEQUENCE_LENGTH\n)\n\n# Define the model block including: inputs, masking, projection and transformer block.\nbody = tr.SequentialBlock(\n    inputs,\n    tr.MLPBlock([D_MODEL]),\n    tr.TransformerBlock(transformer_config, masking=inputs.masking)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defines the evaluation top-N metrics and the cut-offs\nmetrics = [\n    RecallAt(top_ks=[10, 20], labels_onehot=True)\n]\n\n# Define a head related to next item prediction task \n# Important: transformerc4rec v0.1.15\n# NextItemPredictionTask should have parameter [hf_format=True]\nhead = tr.Head(\n    body,\n    tr.NextItemPredictionTask(weight_tying=True, hf_format=True, metrics=metrics),\n    inputs=inputs,\n)\n\n# Get the end-to-end Model class \nmodel = tr.Model(head)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set hyperparameters for training \n\ntrain_args = T4RecTrainingArguments(\n    data_loader_engine='nvtabular', \n    dataloader_drop_last=True,\n    gradient_accumulation_steps=1,\n    per_device_train_batch_size=TRAIN_BATCH_SIZE, \n    per_device_eval_batch_size=EVAL_BATCH_SIZE,\n    output_dir=output_path, \n    learning_rate=0.0005,\n    lr_scheduler_type='cosine', \n    learning_rate_num_cosine_cycles_by_epoch=1,\n    num_train_epochs=1,\n    max_sequence_length=MAX_SEQUENCE_LENGTH, \n    logging_steps=500,\n    save_steps=1000,\n    save_total_limit=5,\n    no_cuda=False,\n    fp16=True,\n    dataloader_num_workers=4,\n    max_steps=MAX_STEPS,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,\n    args=train_args,\n    schema=schema_train,\n    compute_metrics=True,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = \"../input/otto-t4rec-checkpoint\"\ntrainer.load_model_trainer_states_from_checkpoint(checkpoint, model=model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Candidates dictionary from Co-visitation matrix\n\n\"Carts Orders\" Co-visitation Matrix - Type Weighted\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=7\n\n\"Buy2Buy\" Co-visitation Matrix\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=9\n\n\"Clicks\" Co-visitation Matrix - Time Weighted\n\nhttps://www.kaggle.com/code/cdeotte/candidate-rerank-model-lb-0-575?scriptVersionId=111214204&cellId=11","metadata":{}},{"cell_type":"code","source":"# top_15_buy2buy_path = \"../input/candidate-rerank-model-lb-0-575/top_15_buy2buy_v5_0.pqt\"\n# top_15_carts_orders_path = glob.glob(\"../input/candidate-rerank-model-lb-0-575/top_15_carts_orders_v5_*.pqt\")\n# top_20_clicks_path = glob.glob(\"/kaggle/input/candidate-rerank-model-lb-0-575/top_20_clicks_v5_*.pqt\")\n\ntop_20_clicks_path = \"../input/otto-etl-with-co-visitation/top_20_clicks.pkl\"\ntop_15_buy2buy_path = \"../input/otto-etl-with-co-visitation/top_15_buy2buy.pkl\"\ntop_15_carts_orders_path = \"../input/otto-etl-with-co-visitation/top_15_carts_orders.pkl\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pqt_to_dict(df):\n    return df.groupby('aid_x').aid_y.apply(list).to_dict()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# top_20_clicks = pqt_to_dict(pd.read_parquet(top_20_clicks_path))\n# top_15_buy2buy = pqt_to_dict(pd.read_parquet(top_15_buy2buy_path))\n# top_15_carts_orders = pqt_to_dict(pd.read_parquet(top_15_carts_orders_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith open(top_20_clicks_path, \"rb\") as f:\n    top_20_clicks = pickle.load(f)\n\nwith open(top_15_buy2buy_path, \"rb\") as f:\n    top_15_buy2buy = pickle.load(f)\n    \nwith open(top_15_carts_orders_path, \"rb\") as f:\n    top_15_carts_orders = pickle.load(f)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates = np.load(\"../input/otto-etl-with-co-visitation/candidates.npy\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_to_aid = dict(enumerate(candidates, start=2))\naid_to_encoded = {v: k for k, v in encoded_to_aid.items()}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_encode_(aids):\n    return [aid_to_encoded.get(x, 1) for x in aids]\n\ndef to_encode(col):\n    result = []\n    for aids in col:\n        result.append(to_encoded_(aids))\n    return result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aid_to_decode(aid, start_index=2):\n    # index 0: padding, index 1: OOV, index 2: start index\n    if aid >= start_index:\n        result = candidates[aid - start_index]\n    # OOV to candidates[0]\n    elif aid == 1:\n        result = candidates[0]\n    else:\n        result = 0\n        \n    return result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_decode_(aids, start_index=2):\n    # index 0: padding, index 1: OOV, index 2: start index\n    result = []\n    for e in np.nditer(aids):\n        if e >= start_index:\n            decoded = candidates[e - start_index]\n        # OOV to candidates[0]\n        elif e == 1:\n            decoded = candidates[0]\n        else:\n            decoded = 0\n            \n        result.append(decoded)\n    return np.asarray(result)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare DataLoader for validation","metadata":{}},{"cell_type":"code","source":"def zero_padding(x, length):\n    x += [0] * (length - len(x))\n    return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from enum import Enum\nclass LabelTypes(Enum):\n    CLICKS = 1\n    CARTS = 2\n    ORDERS = 3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOP_K = 20\nLABEL_TYPES = [\"clicks\", \"carts\", \"orders\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in LabelTypes:\n    print(k.value)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_dataload = T4RecDataLoader.parse('pyarrow').from_schema(\n    schema_valid,\n    valid_path,\n    EVAL_BATCH_SIZE,\n    max_sequence_length=MAX_SEQUENCE_LENGTH,\n    drop_last=True,\n    shuffle=False,\n#     shuffle_buffer_size=self.args.shuffle_buffer_size,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(columns=[\"session_type\", \"labels\", \"ground_truth\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch = next(iter(eval_dataload))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"['session', 'event_type', 'top_20_clicks-list', 'top_15_buy2buy-list', 'top_15_carts_orders-list', 'aid-list', 'labels_clicks', 'labels_carts', 'labels_orders']","metadata":{}},{"cell_type":"code","source":"for batch in tqdm(eval_dataload):\n\n    top_20_clicks_list = []\n    top_15_buy2buy_list = []\n    top_15_carts_orders_list = []\n\n    sessions = batch[\"session\"].detach().cpu().numpy()\n    labels_gt = {\n        LabelTypes.CLICKS.name: batch[\"labels_clicks\"].detach().cpu().numpy(),\n        LabelTypes.CARTS.name: batch[\"labels_carts\"].detach().cpu().numpy(),\n        LabelTypes.ORDERS.name: batch[\"labels_orders\"].detach().cpu().numpy(),\n    }\n\n    for idx, aids in enumerate(batch[\"aid-list\"]):\n        last_nonzero_index = torch.max(torch.nonzero(aids)).numpy()\n        \n        last_aid = aids[last_nonzero_index].numel()\n        last_aid_raw = aid_to_decode(last_aid)\n\n        top_20_clicks_list.append(zero_padding(to_encode_(top_20_clicks[last_aid_raw]), 500))\n        top_15_buy2buy_list.append(zero_padding(to_encode_(top_15_buy2buy[last_aid_raw]), 500))\n        top_15_carts_orders_list.append(zero_padding(to_encode_(top_15_carts_orders[last_aid_raw]),500))\n\n    batch[\"top_20_clicks-list\"] = torch.as_tensor(top_20_clicks_list)\n    batch[\"top_15_buy2buy-list\"] = torch.as_tensor(top_15_buy2buy_list)\n    batch[\"top_15_carts_orders-list\"] = torch.as_tensor(top_15_carts_orders_list)\n    \n    for label_type in LabelTypes:\n        event_type_list = [label_type.value] * len(batch[\"event_type\"])\n\n        batch[\"event_type\"] = torch.as_tensor(event_type_list)\n\n#         print(\"event_type_list\", event_type_list)\n#         print(batch[\"session\"].shape)\n#         print(batch[\"event_type\"].shape)\n#         print(batch[\"top_20_clicks-list\"].shape)\n#         print(batch[\"top_15_buy2buy-list\"].shape)\n#         print(batch[\"top_15_carts_orders-list\"].shape)\n#         print(batch[\"aid-list\"].shape)\n#         print(batch[\"labels_clicks\"].shape)\n#         print(batch[\"labels_carts\"].shape)\n#         print(batch[\"labels_orders\"].shape)\n\n        for k, v in batch.items():\n            batch[k] = v.to(\"cuda:0\")\n        \n        #     batch[\"aid-list\"].is_cuda        \n        \n        output = model(batch, training=False)\n\n        predictions = output[\"predictions\"].detach().cpu().numpy()\n        top_preds_ = np.argpartition(predictions, -TOP_K, axis=1)[:, -TOP_K:]\n        top_preds = [to_decode_(x) for x in top_preds_]\n        # numpy ndarray element to string elements with white space\n#         top_preds = [(\" \").join(map(str, x.tolist())) for x in top_preds]\n\n        session_types = [str(s) + \"_\" + label_type.name.lower() for s in sessions]\n        \n        df = pd.concat(\n            [\n                df,\n                pd.DataFrame(\n                    {\n                        \"session_type\": session_types,\n                        \"labels\": top_preds,\n                        \"ground_truth\": list(labels_gt[label_type.name]),\n                    }\n                )\n            ], axis=0\n        )\n    \n    if DEBUG:\n        break\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_zero(input):\n    if isinstance(input, list) or isinstance(input, np.ndarray):\n        input = set(input)\n        input.discard(0)\n        input = list(input)\n    elif isinstance(input, int):\n        input = [input]\n    return input\n        \n\nweights = {\"clicks\": 0.10, \"carts\": 0.30, \"orders\": 0.60}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"ground_truth\"] = df[\"ground_truth\"].map(remove_zero)\ndf[\"gt_count\"] = df[\"ground_truth\"].map(len)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df[\"ground_truth\"].map(len) != 0]\ndf = df[df[\"ground_truth\"].map(lambda x:x[0]) != 0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"hits\"]= df.apply(lambda z: len(set(z.ground_truth).intersection(set(z.labels))), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_clicks = df[df.session_type.str.contains(\"clicks\")]\ndf_carts = df[df.session_type.str.contains(\"carts\")]\ndf_orders = df[df.session_type.str.contains(\"orders\")]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recall_clicks = df_clicks[\"hits\"].sum() / df_clicks[\"gt_count\"].sum()\nrecall_carts = df_carts[\"hits\"].sum() / df_carts[\"gt_count\"].sum()\nrecall_orders = df_clicks[\"hits\"].sum() / df_orders[\"gt_count\"].sum()\n\nscore = weights[\"clicks\"] * recall_clicks + \\\n        weights[\"carts\"] * recall_carts + \\\n        weights[\"orders\"] * recall_orders","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Click   Recall = \", recall_clicks)\nprint(\"Carts   Recall = \", recall_carts)\nprint(\"Orders  Recall = \", recall_orders)\nprint(\"===========================\")\nprint(\"Overall Reacll = \", score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}