{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nvtabular==1.3.3","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-31T02:49:55.511675Z","iopub.execute_input":"2023-01-31T02:49:55.512259Z","iopub.status.idle":"2023-01-31T02:52:17.765752Z","shell.execute_reply.started":"2023-01-31T02:49:55.512217Z","shell.execute_reply":"2023-01-31T02:52:17.764237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\nimport shutil\n\nimport dask.dataframe as dd\nimport numpy as np\nimport pandas as pd\nimport nvtabular as nvt\nfrom merlin.dag import ColumnSelector\nfrom merlin.schema import Tags","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:17.768960Z","iopub.execute_input":"2023-01-31T02:52:17.769538Z","iopub.status.idle":"2023-01-31T02:52:20.239256Z","shell.execute_reply.started":"2023-01-31T02:52:17.769479Z","shell.execute_reply":"2023-01-31T02:52:20.238082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:20.240858Z","iopub.execute_input":"2023-01-31T02:52:20.241353Z","iopub.status.idle":"2023-01-31T02:52:20.246800Z","shell.execute_reply.started":"2023-01-31T02:52:20.241304Z","shell.execute_reply":"2023-01-31T02:52:20.245795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_working_dir():\n    paths = [\n        \"../working/train\",\n        \"../working/valid\",\n        \"../working/test\"\n    ]\n    \n    for p in paths:\n        if os.path.exists(p):\n            shutil.rmtree(p)\n\n    print(os.listdir(\"../working\"))\n    ","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:20.249049Z","iopub.execute_input":"2023-01-31T02:52:20.250217Z","iopub.status.idle":"2023-01-31T02:52:20.263750Z","shell.execute_reply.started":"2023-01-31T02:52:20.250160Z","shell.execute_reply":"2023-01-31T02:52:20.262217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_working_dir()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:20.265250Z","iopub.execute_input":"2023-01-31T02:52:20.265753Z","iopub.status.idle":"2023-01-31T02:52:20.277704Z","shell.execute_reply.started":"2023-01-31T02:52:20.265704Z","shell.execute_reply":"2023-01-31T02:52:20.276078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NVTabular ETL","metadata":{}},{"cell_type":"code","source":"if DEBUG:\n    trainset = \"../input/otto-etl-with-co-visitation/train_0.parquet\"\n    validset = \"../input/otto-etl-with-co-visitation/valid_0.parquet\"\n    testset = \"../input/otto-etl-with-co-visitation/test_0.parquet\"\n    \n    train_output_files=None\n    valid_output_files=None\n    test_output_files=None\nelse:\n    trainset = glob.glob(\"../input/otto-etl-with-co-visitation/train_*.parquet\")\n    validset = glob.glob(\"../input/otto-etl-with-co-visitation/valid_*.parquet\")\n    testset = glob.glob(\"../input/otto-etl-with-co-visitation/test_*.parquet\")\n    \n    train_output_files=60\n    valid_output_files=10\n    test_output_files=10","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:21.511649Z","iopub.execute_input":"2023-01-31T02:52:21.512029Z","iopub.status.idle":"2023-01-31T02:52:21.519667Z","shell.execute_reply.started":"2023-01-31T02:52:21.511998Z","shell.execute_reply":"2023-01-31T02:52:21.518380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dd.read_parquet(trainset)\nvalid_df = dd.read_parquet(validset)\ntest_df = dd.read_parquet(testset)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:21.521112Z","iopub.execute_input":"2023-01-31T02:52:21.521523Z","iopub.status.idle":"2023-01-31T02:52:21.671764Z","shell.execute_reply.started":"2023-01-31T02:52:21.521480Z","shell.execute_reply":"2023-01-31T02:52:21.670049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates_path = \"../input/otto-candidates-orders/candidates_orders.npy\"\ncandidates = np.load(candidates_path)\nlen(candidates)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T03:01:11.807407Z","iopub.execute_input":"2023-01-31T03:01:11.808539Z","iopub.status.idle":"2023-01-31T03:01:11.823112Z","shell.execute_reply.started":"2023-01-31T03:01:11.808489Z","shell.execute_reply":"2023-01-31T03:01:11.821956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_path = \"../working/\"\n%cp {candidates_path} {output_path}","metadata":{"execution":{"iopub.status.busy":"2023-01-31T03:01:47.524807Z","iopub.execute_input":"2023-01-31T03:01:47.525263Z","iopub.status.idle":"2023-01-31T03:01:48.715629Z","shell.execute_reply.started":"2023-01-31T03:01:47.525225Z","shell.execute_reply":"2023-01-31T03:01:48.714051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encode_to_aid = dict(enumerate(candidates, start=2))\naid_to_encode = {v: k for k, v in encode_to_aid.items()}","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:52:38.242071Z","iopub.execute_input":"2023-01-31T02:52:38.242684Z","iopub.status.idle":"2023-01-31T02:52:38.608870Z","shell.execute_reply.started":"2023-01-31T02:52:38.242477Z","shell.execute_reply":"2023-01-31T02:52:38.607636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AID_DOMAIN_LAST = list(aid_to_encode.values())[-1]\nprint(AID_DOMAIN_LAST)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:58:58.244009Z","iopub.execute_input":"2023-01-31T02:58:58.244416Z","iopub.status.idle":"2023-01-31T02:58:58.273563Z","shell.execute_reply.started":"2023-01-31T02:58:58.244384Z","shell.execute_reply":"2023-01-31T02:58:58.271408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_encode_(aids):\n    return [aid_to_encode.get(x, 1) for x in aids]\n\ndef to_encode(col):\n    result = []\n    for aids in col:\n        result.append(to_encode_(aids))\n    return result","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:58:27.994939Z","iopub.execute_input":"2023-01-31T02:58:27.995390Z","iopub.status.idle":"2023-01-31T02:58:28.002106Z","shell.execute_reply.started":"2023-01-31T02:58:27.995351Z","shell.execute_reply":"2023-01-31T02:58:28.000820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:58:28.141505Z","iopub.execute_input":"2023-01-31T02:58:28.142528Z","iopub.status.idle":"2023-01-31T02:58:30.424046Z","shell.execute_reply.started":"2023-01-31T02:58:28.142473Z","shell.execute_reply":"2023-01-31T02:58:30.422842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:58:30.426896Z","iopub.execute_input":"2023-01-31T02:58:30.427410Z","iopub.status.idle":"2023-01-31T02:58:31.294003Z","shell.execute_reply.started":"2023-01-31T02:58:30.427360Z","shell.execute_reply":"2023-01-31T02:58:31.293001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:58:31.295555Z","iopub.execute_input":"2023-01-31T02:58:31.296116Z","iopub.status.idle":"2023-01-31T02:58:32.230858Z","shell.execute_reply.started":"2023-01-31T02:58:31.296082Z","shell.execute_reply":"2023-01-31T02:58:32.229937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Number of products ordered vs. number of products clicked\n\nhttps://www.kaggle.com/code/parthpankajtiwary/otto-eda-understanding-users-and-events?scriptVersionId=111667650&cellId=50","metadata":{}},{"cell_type":"code","source":"session_features = (\n    ColumnSelector([\"session\"]) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.CONTEXT],\n        properties={\n            \"value_count\": {\n                \"min\": 1,\n                \"max\": 1,\n            },\n            \"int_domain\": {\n                \"name\": \"event_type\",\n                \"min\": 0,\n                \"max\": 12899778,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)\n\n\nevent_features = (\n    ColumnSelector([\"event_type\"]) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.CONTEXT],\n        properties={\n            \"value_count\": {\n                \"min\": 1,\n                \"max\": 1,\n            },\n            \"int_domain\": {\n                \"name\": \"event_type\",\n                \"min\": 1,\n                \"max\": 3,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)\n\naid_features = (\n    ColumnSelector([\"aid-list\"]) >>\n    nvt.ops.LambdaOp(to_encode) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.ITEM_ID],\n        properties={\n            \"value_count\": {\n                \"min\": 1,\n                \"max\": 500,\n            },\n            \"int_domain\": {\n                \"name\": \"aid-list\",\n                \"min\": 1,\n                \"max\": AID_DOMAIN_LAST,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)\n\nlabels_clicks_features = (\n    ColumnSelector([\"labels_clicks\"]) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.CONTEXT],\n        properties={\n            \"value_count\": {\n                \"min\":1,\n                \"max\":1,\n            },\n            \"int_domain\": {\n                \"name\": \"labels_clicks\",\n                \"min\": 1,\n                \"max\": 1855603,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)\n\nlabels_carts_features = (\n    ColumnSelector([\"labels_carts\"]) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.CONTEXT],\n        properties={\n            \"value_count\": {\n                \"min\":1,\n                \"max\":150,\n            },\n            \"int_domain\": {\n                \"name\": \"labels_carts\",\n                \"min\": 1,\n                \"max\": 1855603,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)\n\nlabels_orders_features = (\n    ColumnSelector([\"labels_orders\"]) >>\n    nvt.ops.AddMetadata(\n        tags=[Tags.CATEGORICAL, Tags.CONTEXT],\n        properties={\n            \"value_count\": {\n                \"min\":1,\n                \"max\":100,\n            },\n            \"int_domain\": {\n                \"name\": \"labels_orders\",\n                \"min\": 1,\n                \"max\": 1855603,\n                \"is_categorical\": True,\n            }\n        }\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:05.289646Z","iopub.execute_input":"2023-01-31T02:59:05.290052Z","iopub.status.idle":"2023-01-31T02:59:05.308947Z","shell.execute_reply.started":"2023-01-31T02:59:05.290022Z","shell.execute_reply":"2023-01-31T02:59:05.307505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_features = (\n    session_features +\n    event_features +\n    aid_features +\n    labels_clicks_features +\n    labels_carts_features +\n    labels_orders_features\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:16.255140Z","iopub.execute_input":"2023-01-31T02:59:16.255561Z","iopub.status.idle":"2023-01-31T02:59:16.261377Z","shell.execute_reply.started":"2023-01-31T02:59:16.255526Z","shell.execute_reply":"2023-01-31T02:59:16.260477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = nvt.Dataset(train_df, cpu=True)\nvalidset = nvt.Dataset(valid_df, cpu=True)\ntestset = nvt.Dataset(test_df, cpu=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:16.429929Z","iopub.execute_input":"2023-01-31T02:59:16.431056Z","iopub.status.idle":"2023-01-31T02:59:18.736323Z","shell.execute_reply.started":"2023-01-31T02:59:16.431013Z","shell.execute_reply":"2023-01-31T02:59:18.735095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"workflow = nvt.Workflow(all_features)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:18.738645Z","iopub.execute_input":"2023-01-31T02:59:18.739003Z","iopub.status.idle":"2023-01-31T02:59:18.744631Z","shell.execute_reply.started":"2023-01-31T02:59:18.738972Z","shell.execute_reply":"2023-01-31T02:59:18.742987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_output_path = \"../working/train\"\nvalid_output_path = \"../working/valid\"\ntest_output_path = \"../working/test\"","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:18.746325Z","iopub.execute_input":"2023-01-31T02:59:18.746736Z","iopub.status.idle":"2023-01-31T02:59:18.759257Z","shell.execute_reply.started":"2023-01-31T02:59:18.746703Z","shell.execute_reply":"2023-01-31T02:59:18.758365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nworkflow.fit(trainset)\nworkflow.transform(trainset).to_parquet(output_path=train_output_path, output_files=train_output_files)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:18.761228Z","iopub.execute_input":"2023-01-31T02:59:18.762047Z","iopub.status.idle":"2023-01-31T02:59:27.920744Z","shell.execute_reply.started":"2023-01-31T02:59:18.762010Z","shell.execute_reply":"2023-01-31T02:59:27.919551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nworkflow.fit(validset)\nworkflow.transform(validset).to_parquet(output_path=valid_output_path, output_files=valid_output_files)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:27.923700Z","iopub.execute_input":"2023-01-31T02:59:27.924135Z","iopub.status.idle":"2023-01-31T02:59:30.645977Z","shell.execute_reply.started":"2023-01-31T02:59:27.924099Z","shell.execute_reply":"2023-01-31T02:59:30.644691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nworkflow.fit(testset)\nworkflow.transform(testset).to_parquet(output_path=test_output_path, output_files=test_output_files)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:30.647779Z","iopub.execute_input":"2023-01-31T02:59:30.648143Z","iopub.status.idle":"2023-01-31T02:59:33.526973Z","shell.execute_reply.started":"2023-01-31T02:59:30.648111Z","shell.execute_reply":"2023-01-31T02:59:33.525664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"workflow.output_schema","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:33.529314Z","iopub.execute_input":"2023-01-31T02:59:33.529784Z","iopub.status.idle":"2023-01-31T02:59:33.546418Z","shell.execute_reply.started":"2023-01-31T02:59:33.529748Z","shell.execute_reply":"2023-01-31T02:59:33.545189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"workflow.output_schema.column_names","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:33.548002Z","iopub.execute_input":"2023-01-31T02:59:33.548478Z","iopub.status.idle":"2023-01-31T02:59:33.558574Z","shell.execute_reply.started":"2023-01-31T02:59:33.548431Z","shell.execute_reply":"2023-01-31T02:59:33.557499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat ../working/train/schema.pbtxt","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:33.560096Z","iopub.execute_input":"2023-01-31T02:59:33.561346Z","iopub.status.idle":"2023-01-31T02:59:34.735259Z","shell.execute_reply.started":"2023-01-31T02:59:33.561306Z","shell.execute_reply":"2023-01-31T02:59:34.733405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.read_parquet(train_output_path).head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:34.737924Z","iopub.execute_input":"2023-01-31T02:59:34.738467Z","iopub.status.idle":"2023-01-31T02:59:35.138842Z","shell.execute_reply.started":"2023-01-31T02:59:34.738411Z","shell.execute_reply":"2023-01-31T02:59:35.137667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.read_parquet(valid_output_path).head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:35.140633Z","iopub.execute_input":"2023-01-31T02:59:35.140966Z","iopub.status.idle":"2023-01-31T02:59:35.321302Z","shell.execute_reply.started":"2023-01-31T02:59:35.140936Z","shell.execute_reply":"2023-01-31T02:59:35.320121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.read_parquet(test_output_path).head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T02:59:35.323053Z","iopub.execute_input":"2023-01-31T02:59:35.323828Z","iopub.status.idle":"2023-01-31T02:59:35.486315Z","shell.execute_reply.started":"2023-01-31T02:59:35.323782Z","shell.execute_reply":"2023-01-31T02:59:35.485252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}