{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install transformers4rec[pytorch,nvtabular,dataloader]\n# !pip install merlin-dataloader>=0.0.2\n\n!pip install transformers4rec[pytorch,nvtabular]\n!pip install -U nvtabular==1.3.3\n!pip install -U pytorch_lightning==1.8.0.post1\n!pip install torchmetrics==0.10.0\n!pip install neptune-client","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\nimport warnings\n\nimport torch\n\nwarnings.simplefilter(action=\"ignore\", category=FutureWarning)\nwarnings.simplefilter(action=\"ignore\", category=UserWarning)\n\nfrom merlin_standard_lib import Schema, Tag\nfrom transformers4rec import torch as tr\nfrom transformers4rec.torch import Trainer\nfrom transformers4rec.torch.ranking_metric import RecallAt\nfrom transformers4rec.config.trainer import T4RecTrainingArguments","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:37.109864Z","iopub.execute_input":"2023-01-24T03:57:37.110288Z","iopub.status.idle":"2023-01-24T03:57:41.381144Z","shell.execute_reply.started":"2023-01-24T03:57:37.110204Z","shell.execute_reply":"2023-01-24T03:57:41.380094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Secret for Neptune logging","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"neptune_api_token\")\n\nos.environ[\"NEPTUNE_PROJECT\"] = \"sunghyun.jun/otto-recsys\"\nos.environ[\"NEPTUNE_API_TOKEN\"] = secret_value_0","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.383113Z","iopub.execute_input":"2023-01-24T03:57:41.383436Z","iopub.status.idle":"2023-01-24T03:57:41.692829Z","shell.execute_reply.started":"2023-01-24T03:57:41.383408Z","shell.execute_reply":"2023-01-24T03:57:41.691781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nRESUME = False","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.694339Z","iopub.execute_input":"2023-01-24T03:57:41.695414Z","iopub.status.idle":"2023-01-24T03:57:41.700517Z","shell.execute_reply.started":"2023-01-24T03:57:41.695376Z","shell.execute_reply":"2023-01-24T03:57:41.699249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 500\nD_MODEL = 64\n# D_MODEL = 320\n\nif DEBUG:\n    MAX_STEPS = 2000\nelse:\n    MAX_STEPS = -1","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.703512Z","iopub.execute_input":"2023-01-24T03:57:41.703995Z","iopub.status.idle":"2023-01-24T03:57:41.710569Z","shell.execute_reply.started":"2023-01-24T03:57:41.703959Z","shell.execute_reply":"2023-01-24T03:57:41.709698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We don't use it, instead create schema proto manually\n# train_schema_path = \"../input/otto-etl-nvtabular-cpu/train/schema.pbtxt\"\n\nif DEBUG:\n#     train_path = \"../input/otto-etl-nvtabular-cpu/train/part_0.parquet\"\n    train_path = [\n        \"../input/otto-etl-nvtabular-cpu/train/part_0.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_1.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_2.parquet\",\n        \"../input/otto-etl-nvtabular-cpu/train/part_3.parquet\",\n    ]\n    valid_path = \"../input/otto-etl-nvtabular-cpu/valid/part_0.parquet\"\n    test_path = \"../input/otto-etl-nvtabular-cpu/test/part_0.parquet\"\nelse:\n    train_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/train/part_*.parquet\"))\n    valid_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/valid/part_*.parquet\"))\n    test_path = sorted(glob.glob(\"../input/otto-etl-nvtabular-cpu/test/part_*.parquet\"))\n    \noutput_path = \"checkpoint/\"","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.712013Z","iopub.execute_input":"2023-01-24T03:57:41.712525Z","iopub.status.idle":"2023-01-24T03:57:41.725817Z","shell.execute_reply.started":"2023-01-24T03:57:41.712490Z","shell.execute_reply":"2023-01-24T03:57:41.724887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create schema proto manually","metadata":{}},{"cell_type":"code","source":"%%writefile schema.pb\n\nfeature {\n  name: \"event_type\"\n  type: INT\n  int_domain {\n    name: \"event_type\"\n    min: 1\n    max: 3\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"categorical\"\n#     extra_metadata {\n#       type_url: \"type.googleapis.com/google.protobuf.Struct\"\n#       value: \"\\n\\034\\n\\017dtype_item_size\\022\\t\\021\\000\\000\\000\\000\\000\\000P@\\n`\\n\\nint_domain\\022R*P\\n\\020\\n\\003max\\022\\t\\021\\000\\000\\000\\000\\000\\000\\010@\\n\\024\\n\\016is_categorical\\022\\002 \\001\\n\\020\\n\\003min\\022\\t\\021\\000\\000\\000\\000\\000\\000\\360?\\n\\024\\n\\004name\\022\\014\\032\\nevent_type\\n\\017\\n\\tis_ragged\\022\\002 \\000\\n\\r\\n\\007is_list\\022\\002 \\000\"\n#     }\n  }\n}\nfeature {\n  name: \"top_20_clicks-list\"\n  value_count {\n    min: 1\n    max: 20\n  }\n  type: INT\n  int_domain {\n    name: \"top_20_clicks-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n#     extra_metadata {\n#       type_url: \"type.googleapis.com/google.protobuf.Struct\"\n#       value: \"\\n\\017\\n\\tis_ragged\\022\\002 \\001\\n\\r\\n\\007is_list\\022\\002 \\001\\nh\\n\\nint_domain\\022Z*X\\n\\034\\n\\004name\\022\\024\\032\\022top_20_clicks-list\\n\\020\\n\\003max\\022\\t\\021\\000\\000\\000\\000\\300\\207\\323@\\n\\020\\n\\003min\\022\\t\\021\\000\\000\\000\\000\\000\\000\\360?\\n\\024\\n\\016is_categorical\\022\\002 \\001\\n\\034\\n\\017dtype_item_size\\022\\t\\021\\000\\000\\000\\000\\000\\000P@\"\n#     }\n  }\n}\nfeature {\n  name: \"top_15_buy2buy-list\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_buy2buy-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n#     extra_metadata {\n#       type_url: \"type.googleapis.com/google.protobuf.Struct\"\n#       value: \"\\n\\034\\n\\017dtype_item_size\\022\\t\\021\\000\\000\\000\\000\\000\\000P@\\n\\r\\n\\007is_list\\022\\002 \\001\\ni\\n\\nint_domain\\022[*Y\\n\\024\\n\\016is_categorical\\022\\002 \\001\\n\\020\\n\\003max\\022\\t\\021\\000\\000\\000\\000\\300\\207\\323@\\n\\020\\n\\003min\\022\\t\\021\\000\\000\\000\\000\\000\\000\\360?\\n\\035\\n\\004name\\022\\025\\032\\023top_15_buy2buy-list\\n\\017\\n\\tis_ragged\\022\\002 \\001\"\n#     }\n  }\n}\nfeature {\n  name: \"top_15_carts_orders-list\"\n  value_count {\n    min: 1\n    max: 15\n  }\n  type: INT\n  int_domain {\n    name: \"top_15_carts_orders-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"context\"\n    tag: \"list\"\n    tag: \"categorical\"\n#     extra_metadata {\n#       type_url: \"type.googleapis.com/google.protobuf.Struct\"\n#       value: \"\\nn\\n\\nint_domain\\022`*^\\n\\020\\n\\003min\\022\\t\\021\\000\\000\\000\\000\\000\\000\\360?\\n\\\"\\n\\004name\\022\\032\\032\\030top_15_carts_orders-list\\n\\020\\n\\003max\\022\\t\\021\\000\\000\\000\\000\\300\\207\\323@\\n\\024\\n\\016is_categorical\\022\\002 \\001\\n\\017\\n\\tis_ragged\\022\\002 \\001\\n\\r\\n\\007is_list\\022\\002 \\001\\n\\034\\n\\017dtype_item_size\\022\\t\\021\\000\\000\\000\\000\\000\\000P@\"\n#     }\n  }\n}\nfeature {\n  name: \"aid-list\"\n  value_count {\n    min: 1\n    max: 500\n  }\n  type: INT\n  int_domain {\n    name: \"aid-list\"\n    min: 1\n    max: 19999\n    is_categorical: true\n  }\n  annotation {\n    tag: \"item\"\n    tag: \"categorical\"\n    tag: \"list\"\n    tag: \"item_id\"\n#     extra_metadata {\n#       type_url: \"type.googleapis.com/google.protobuf.Struct\"\n#       value: \"\\n\\017\\n\\tis_ragged\\022\\002 \\001\\n^\\n\\nint_domain\\022P*N\\n\\020\\n\\003max\\022\\t\\021\\000\\000\\000\\000\\300\\207\\323@\\n\\024\\n\\016is_categorical\\022\\002 \\001\\n\\020\\n\\003min\\022\\t\\021\\000\\000\\000\\000\\000\\000\\360?\\n\\022\\n\\004name\\022\\n\\032\\010aid-list\\n\\r\\n\\007is_list\\022\\002 \\001\\n\\034\\n\\017dtype_item_size\\022\\t\\021\\000\\000\\000\\000\\000\\000P@\"\n#     }\n  }\n}","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.727455Z","iopub.execute_input":"2023-01-24T03:57:41.727978Z","iopub.status.idle":"2023-01-24T03:57:41.737870Z","shell.execute_reply.started":"2023-01-24T03:57:41.727944Z","shell.execute_reply":"2023-01-24T03:57:41.736819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# schema = Schema().from_proto_text(train_schema_path)\nschema = Schema().from_proto_text(\"schema.pb\")","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.739211Z","iopub.execute_input":"2023-01-24T03:57:41.740126Z","iopub.status.idle":"2023-01-24T03:57:41.765226Z","shell.execute_reply.started":"2023-01-24T03:57:41.740091Z","shell.execute_reply":"2023-01-24T03:57:41.764349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = tr.TabularSequenceFeatures.from_schema(\n    schema,\n    max_sequence_length=MAX_SEQUENCE_LENGTH,\n#     aggregation=\"concat\",\n    d_output=D_MODEL,\n    masking=\"mlm\",\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.766562Z","iopub.execute_input":"2023-01-24T03:57:41.766903Z","iopub.status.idle":"2023-01-24T03:57:41.853951Z","shell.execute_reply.started":"2023-01-24T03:57:41.766870Z","shell.execute_reply":"2023-01-24T03:57:41.852880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:41.907138Z","iopub.execute_input":"2023-01-24T03:57:41.908784Z","iopub.status.idle":"2023-01-24T03:57:41.919378Z","shell.execute_reply.started":"2023-01-24T03:57:41.908737Z","shell.execute_reply":"2023-01-24T03:57:41.918349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define XLNetConfig class and set default parameters for HF XLNet config  \ntransformer_config = tr.XLNetConfig.build(\n    d_model=D_MODEL, n_head=4, n_layer=2, total_seq_length=MAX_SEQUENCE_LENGTH\n)\n\n# Define the model block including: inputs, masking, projection and transformer block.\nbody = tr.SequentialBlock(\n    inputs,\n    tr.MLPBlock([D_MODEL]),\n    tr.TransformerBlock(transformer_config, masking=inputs.masking)\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:42.138726Z","iopub.execute_input":"2023-01-24T03:57:42.139110Z","iopub.status.idle":"2023-01-24T03:57:42.159490Z","shell.execute_reply.started":"2023-01-24T03:57:42.139079Z","shell.execute_reply":"2023-01-24T03:57:42.158378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defines the evaluation top-N metrics and the cut-offs\nmetrics = [\n    RecallAt(top_ks=[10, 20], labels_onehot=True)\n]\n\n# Define a head related to next item prediction task \n# Important: transformerc4rec v0.1.15\n# NextItemPredictionTask should have parameter [hf_format=True]\nhead = tr.Head(\n    body,\n    tr.NextItemPredictionTask(weight_tying=True, hf_format=True, metrics=metrics),\n    inputs=inputs,\n)\n\n# Get the end-to-end Model class \nmodel = tr.Model(head)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:42.365145Z","iopub.execute_input":"2023-01-24T03:57:42.365742Z","iopub.status.idle":"2023-01-24T03:57:42.374766Z","shell.execute_reply.started":"2023-01-24T03:57:42.365705Z","shell.execute_reply":"2023-01-24T03:57:42.373539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set hyperparameters for training \n\ntrain_args = T4RecTrainingArguments(\n    data_loader_engine='nvtabular', \n    dataloader_drop_last=True,\n    gradient_accumulation_steps=1,\n    per_device_train_batch_size=128, \n    per_device_eval_batch_size=128,\n    output_dir=output_path, \n    learning_rate=0.0005,\n    lr_scheduler_type='cosine', \n    learning_rate_num_cosine_cycles_by_epoch=1,\n    num_train_epochs=1,\n    max_sequence_length=MAX_SEQUENCE_LENGTH, \n    report_to=\"neptune\",\n    logging_steps=500,\n    save_steps=1000,\n    save_total_limit=5,\n    no_cuda=False,\n    fp16=True,\n    dataloader_num_workers=4,\n    max_steps=MAX_STEPS,\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:43.030148Z","iopub.execute_input":"2023-01-24T03:57:43.031409Z","iopub.status.idle":"2023-01-24T03:57:43.040015Z","shell.execute_reply.started":"2023-01-24T03:57:43.031361Z","shell.execute_reply":"2023-01-24T03:57:43.038957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model=model,\n    args=train_args,\n    schema=schema,\n    compute_metrics=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:43.845965Z","iopub.execute_input":"2023-01-24T03:57:43.846359Z","iopub.status.idle":"2023-01-24T03:57:46.818834Z","shell.execute_reply.started":"2023-01-24T03:57:43.846328Z","shell.execute_reply":"2023-01-24T03:57:46.817740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train_dataset_or_path = train_path\ntrainer.eval_dataset_or_path = valid_path","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:46.821100Z","iopub.execute_input":"2023-01-24T03:57:46.821599Z","iopub.status.idle":"2023-01-24T03:57:46.826560Z","shell.execute_reply.started":"2023-01-24T03:57:46.821550Z","shell.execute_reply":"2023-01-24T03:57:46.825344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train(resume_from_checkpoint = RESUME)","metadata":{"execution":{"iopub.status.busy":"2023-01-24T03:57:46.828210Z","iopub.execute_input":"2023-01-24T03:57:46.828924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer._save_model_and_checkpoint(save_model_class=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metrics = trainer.evaluate(eval_dataset=valid_path, metric_key_prefix='eval')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in sorted(train_metrics.keys()):\n    print(\" %s = %s\" % (key, str(train_metrics[key])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}